maxframe 2.0.0b2__cp37-cp37m-win32.whl → 2.3.0rc1__cp37-cp37m-win32.whl

maxframe/codegen/spe/tensor/tests/test_misc.py

@@ -15,7 +15,11 @@
 from ..... import tensor as mt
 from ...core import SPECodeContext
 from ..misc import (
+    TensorCopyToAdapter,
+    TensorGetShapeAdapter,
+    TensorInsertAdapter,
     TensorIsInAdapter,
+    TensorSplitAdapter,
     TensorSqueezeAdapter,
     TensorTransposeAdapter,
     TensorUniqueAdapter,
@@ -40,7 +44,7 @@ def test_transpose():
     adapter = TensorTransposeAdapter()
     context = SPECodeContext()
     results = adapter.generate_code(result.op, context)
-    expected_results = ["var_1 = np.transpose(var_0, axes=[1, 0, 2])"]
+    expected_results = ["var_0 = np.transpose(var_1, axes=[1, 0, 2])"]
     assert results == expected_results
 
 
@@ -50,7 +54,7 @@ def test_squeeze():
     adapter = TensorSqueezeAdapter()
     context = SPECodeContext()
     results = adapter.generate_code(result.op, context)
-    expected_results = ["var_1 = np.squeeze(var_0, axis=(0,))"]
+    expected_results = ["var_0 = np.squeeze(var_1, axis=(0,))"]
     assert results == expected_results
 
 
@@ -92,3 +96,49 @@ def test_unique():
         "np.unique(var_0, return_index=True, return_inverse=True, return_counts=True, axis=0)"
     ]
     assert results == expected_results
+
+
+def test_insert():
+    result = mt.insert(mt.array([[1, 1], [2, 2], [3, 3]]), 1, 5)
+
+    adapter = TensorInsertAdapter()
+    context = SPECodeContext()
+    results = adapter.generate_code(result.op, context)
+    expected_results = ["var_0 = np.insert(var_1, 1, values=5)"]
+    assert results == expected_results
+
+
+def test_split():
+    res_tp = mt.split(mt.array([1, 2, 3, 4, 5, 6]), 3)
+
+    adapter = TensorSplitAdapter()
+    context = SPECodeContext()
+    results = adapter.generate_code(res_tp[0].op, context)
+    expected_results = ["var_1, var_2, var_3 = np.split(var_0, 3 axis=0)"]
+    assert results == expected_results
+
+
+def test_get_shape():
+    inp = mt.array([[1, 1], [2, 2], [3, 3]])
+    res = mt.shape(inp[:, inp[0, :] == 1])
+
+    adapter = TensorGetShapeAdapter()
+    context = SPECodeContext()
+    results = adapter.generate_code(res[0].op, context)
+    expected_results = ["var_1, var_2 = np.shape(var_0)"]
+    assert results == expected_results
+
+
+def test_copyto():
+    dest = mt.array([[1, 1], [2, 2], [3, 3]])
+    src = mt.array([[1, 1], [2, 2], [3, 4]])
+    mt.copyto(dest, src)
+
+    adapter = TensorCopyToAdapter()
+    context = SPECodeContext()
+    results = adapter.generate_code(dest.op, context)
+    expected_results = [
+        "var_2 = var_0.copy()",
+        "np.copyto(var_2, var_1, casting='same_kind')",
+    ]
+    assert results == expected_results

maxframe/codegen/spe/tensor/tests/test_spatial.py

@@ -0,0 +1,33 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ..... import tensor as mt
+from ...core import SPECodeContext
+from ..spatial import TensorCDistAdapter
+
+
+def test_cdist():
+    xa = mt.random.rand(100, 10)
+    xb = mt.random.rand(89, 10)
+    VI = mt.random.rand(10, 10)
+    dist = mt.spatial.distance.cdist(xa, xb, metric="mahalanobis", VI=VI)
+
+    adapter = TensorCDistAdapter()
+    context = SPECodeContext()
+    results = adapter.generate_code(dist.op, context)
+    expected_results = [
+        "var_0 = scipy.spatial.distance.cdist("
+        "var_1, var_2, metric='mahalanobis', VI=var_3)"
+    ]
+    assert results == expected_results

maxframe/codegen/spe/tensor/tests/test_statistics.py

@@ -14,7 +14,7 @@
 
 from ..... import tensor as mt
 from ...core import SPECodeContext
-from ..statistics import TensorBinCountAdapter
+from ..statistics import TensorBinCountAdapter, TensorHistogramAdapter
 
 
 def test_bincount():
@@ -27,3 +27,17 @@ def test_bincount():
     results = adapter.generate_code(result.op, context)
     expected_results = ["var_0 = np.bincount(var_1, weights=var_2, minlength=0)"]
     assert results == expected_results
+
+
+def test_histogram():
+    arr = mt.arange(5)
+    result, _bins = mt.histogram(arr, bins=3)
+
+    adapter = TensorHistogramAdapter()
+    context = SPECodeContext()
+    results = adapter.generate_code(result.op, context)
+    expected_results = [
+        "var_1, _ = np.histogram(var_0, bins=var_2, range=None, density=None, "
+        "weights=None)"
+    ]
+    assert results == expected_results

maxframe/codegen/spe/tests/test_spe_codegen.py

@@ -59,12 +59,8 @@ def test_codegen_with_udf(codegen):
 
     udf_1_body = base64.b64encode(pickle.dumps(f1, protocol=pickle.DEFAULT_PROTOCOL))
     udf_2_body = base64.b64encode(pickle.dumps(f2, protocol=pickle.DEFAULT_PROTOCOL))
-    udf_1_value = (
-        f"pickled_data = cloudpickle.loads(base64.b64decode({udf_1_body}), buffers=[])"
-    )
-    udf_2_value = (
-        f"pickled_data = cloudpickle.loads(base64.b64decode({udf_2_body}), buffers=[])"
-    )
+    udf_1_value = f"udf_main_entry = cloudpickle.loads(base64.b64decode({udf_1_body}), buffers=[])"
+    udf_2_value = f"udf_main_entry = cloudpickle.loads(base64.b64decode({udf_2_body}), buffers=[])"
     udf_1 = f"user_udf_f1_{hashlib.md5(udf_1_value.encode('utf-8')).hexdigest()}"
     udf_2 = f"user_udf_f2_{hashlib.md5(udf_2_value.encode('utf-8')).hexdigest()}"
 
@@ -74,9 +70,9 @@ def test_codegen_with_udf(codegen):
     import base64
     import cloudpickle
     {udf_1_value}
-    {udf_1} = pickled_data
+    {udf_1} = udf_main_entry
     {udf_2_value}
-    {udf_2} = pickled_data
+    {udf_2} = udf_main_entry
     if not running:
         raise RuntimeError('CANCELLED')
     var_0 = np.random.rand(1, 3)
@@ -111,9 +107,7 @@ def test_codegen_with_udf_and_args(codegen):
             protocol=pickle.DEFAULT_PROTOCOL,
         )
     )
-    udf_1_value = (
-        f"pickled_data = cloudpickle.loads(base64.b64decode({udf_1_body}), buffers=[])"
-    )
+    udf_1_value = f"udf_main_entry = cloudpickle.loads(base64.b64decode({udf_1_body}), buffers=[])"
     udf_1 = f"user_udf_f1_{hashlib.md5(udf_1_value.encode('utf-8')).hexdigest()}"
 
     expected_contents = f"""
@@ -122,7 +116,7 @@ def test_codegen_with_udf_and_args(codegen):
     import base64
     import cloudpickle
     {udf_1_value}
-    {udf_1} = pickled_data
+    {udf_1} = udf_main_entry
     if not running:
         raise RuntimeError('CANCELLED')
     var_0 = np.random.rand(1, 3)

maxframe/codegen/spe/utils.py

@@ -37,6 +37,8 @@ def build_method_call_adapter(
         def generate_code(self, op: OperatorType, context: SPECodeContext) -> List[str]:
             if source_module in _import_aliases:
                 context.register_import(_import_aliases[source_module], source_module)
+            elif source_module:
+                context.register_import(source_module)
 
             input_var_name = source_module or context.get_input_tileable_variable(
                 op.inputs[0]

maxframe/config/config.py

@@ -18,7 +18,7 @@ import os
 import traceback
 import warnings
 from copy import deepcopy
-from typing import Any, Dict, Optional, Union
+from typing import Any, Callable, Dict, Optional, Union
 
 from odps.lib import tzlocal
 
@@ -33,13 +33,17 @@ from ..utils import get_python_tag
 from .validators import (
     ValidatorType,
     all_validator,
+    dtype_backend_validator,
     is_all_dict_keys_in,
     is_bool,
     is_dict,
     is_float,
+    is_great_than,
     is_in,
     is_integer,
+    is_less_than_or_equal_to,
     is_non_negative_integer,
+    is_notnull,
     is_null,
     is_numeric,
     is_string,
@@ -59,6 +63,7 @@ _DEFAULT_TASK_START_TIMEOUT = 60
 _DEFAULT_TASK_RESTART_TIMEOUT = 300
 _DEFAULT_LOGVIEW_HOURS = 24 * 30
 _DEFAULT_FUNCTION_RUNNING_OPTIONS = {"cpu": 1, "memory": "4GiB", "gpu": 0}
+_DEFAULT_MAX_MEMORY_CPU_RATIO = 12
 
 
 class OptionError(Exception):
@@ -66,11 +71,19 @@ class OptionError(Exception):
 
 
 class Redirection:
-    def __init__(self, item: str, warn: Optional[str] = None):
+    def __init__(
+        self,
+        item: str,
+        warn: Optional[str] = None,
+        getter: Callable = None,
+        setter: Callable = None,
+    ):
         self._items = item.split(".")
         self._warn = warn
         self._warned = True
         self._parent = None
+        self._getter = getter
+        self._setter = setter
 
     def bind(self, attr_dict):
         self._parent = attr_dict
@@ -88,6 +101,8 @@ class Redirection:
         conf = self._parent.root
         for it in self._items:
             conf = getattr(conf, it)
+        if callable(self._getter):
+            conf = self._getter(conf)
         return conf
 
     def setvalue(self, value: str, silent: bool = False) -> None:
@@ -97,6 +112,8 @@ class Redirection:
         conf = self._parent.root
         for it in self._items[:-1]:
             conf = getattr(conf, it)
+        if callable(self._setter):
+            value = self._setter(value)
         setattr(conf, self._items[-1], value)
 
 
@@ -251,9 +268,19 @@ class Config:
             self._remote_options.add(option)
 
     def redirect_option(
-        self, option: str, target: str, warn: str = _DEFAULT_REDIRECT_WARN
+        self,
+        option: str,
+        target: str,
+        warn: str = _DEFAULT_REDIRECT_WARN,
+        getter: Callable = None,
+        setter: Callable = None,
     ) -> None:
-        redir = Redirection(target, warn=warn.format(source=option, target=target))
+        redir = Redirection(
+            target,
+            warn=warn.format(source=option, target=target),
+            getter=getter,
+            setter=setter,
+        )
         self.register_option(option, redir)
 
     def unregister_option(self, option: str) -> None:
@@ -315,10 +342,18 @@ class Config:
 
 
 def _get_legal_local_tz_name() -> Optional[str]:
-    """Sometimes we may get illegal tz name from tzlocal.get_localzone()"""
-    tz_name = str(tzlocal.get_localzone())
-    if tz_name not in available_timezones():
-        return None
+    """
+    Sometimes we may get illegal tz name from tzlocal.get_localzone().
+    In some environments we can't get any tz name.
+    """
+    tz_name = None
+    try:
+        tz_name = str(tzlocal.get_localzone())
+        if tz_name not in available_timezones():
+            tz_name = None
+    except:
+        pass
+
     return tz_name
 
 
@@ -400,6 +435,12 @@ default_options.register_option(
 default_options.register_option(
     "session.quota_name", None, validator=is_null | is_string, remote=True
 )
+default_options.register_option(
+    "session.gu_quota_name", None, validator=is_null | is_string, remote=True
+)
+default_options.register_option(
+    "session.region_id", None, validator=is_null | is_string, remote=True
+)
 default_options.register_option(
     "session.enable_schema", None, validator=is_null | is_bool, remote=True
 )
@@ -446,7 +487,15 @@ default_options.register_option(
 )
 
 default_options.register_option("warn_duplicated_execution", False, validator=is_bool)
-default_options.register_option("dataframe.use_arrow_dtype", True, validator=is_bool)
+default_options.register_option(
+    "dataframe.dtype_backend", "numpy", validator=dtype_backend_validator
+)
+default_options.redirect_option(
+    "dataframe.use_arrow_dtype",
+    "dataframe.dtype_backend",
+    getter=lambda x: x == "pyarrow",
+    setter=lambda x: "pyarrow" if x else "numpy",
+)
 default_options.register_option(
     "dataframe.arrow_array.pandas_only", True, validator=is_bool
 )
@@ -464,6 +513,15 @@ default_options.register_option(
     validator=is_dict | is_all_dict_keys_in("cpu", "memory", "gpu"),
 )
 
+default_options.register_option(
+    "function.allowed_max_memory_cpu_ratio",
+    _DEFAULT_MAX_MEMORY_CPU_RATIO,
+    validator=is_integer
+    & is_notnull
+    & is_less_than_or_equal_to(_DEFAULT_MAX_MEMORY_CPU_RATIO)
+    & is_great_than(0),
+)
+
 ################
 # DPE Settings #
 ################
@@ -505,9 +563,15 @@ default_options.register_option(
 assume_finite = os.environ.get("SKLEARN_ASSUME_FINITE")
 if assume_finite is not None:
     assume_finite = bool(assume_finite)
+working_memory = os.environ.get("SKLEARN_WORKING_MEMORY")
+if working_memory is not None:
+    working_memory = int(working_memory)
 default_options.register_option(
     "learn.assume_finite", assume_finite, validator=is_null | is_bool
 )
+default_options.register_option(
+    "learn.working_memory", working_memory, validator=is_null | is_integer
+)
 
 _options_ctx_var = contextvars.ContextVar("_options_ctx_var")
 

maxframe/config/tests/test_validators.py

@@ -14,7 +14,11 @@
 
 import pytest
 
-from ..validators import is_positive_integer, simple_yaml_str_validator
+from ..validators import (
+    is_less_than_or_equal_to,
+    is_positive_integer,
+    simple_yaml_str_validator,
+)
 
 
 @pytest.mark.parametrize("value", ["a", "http://127.0.0.1:1234", "a-b#", "ab_", "123"])
@@ -32,3 +36,11 @@ def test_simple_yaml_str_validator_invalid(value):
 )
 def test_is_positive_integer_validator(value, valid):
     assert is_positive_integer(value) is valid
+
+
+@pytest.mark.parametrize(
+    "value,upper_bound,valid",
+    [(3, 5, True), (5, 5, True), (6, 5, False), (None, None, False), (None, 5, False)],
+)
+def test_is_less_than_or_equal_to_validator(value, upper_bound, valid):
+    assert is_less_than_or_equal_to(upper_bound)(value) is valid

maxframe/config/validators.py

@@ -12,9 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 from typing import Callable
 from urllib.parse import urlparse
 
+from .. import env
+from ..utils import str_to_bool
+
 ValidatorType = Callable[..., bool]
 
 
@@ -43,13 +47,22 @@ class Validator:
     def __or__(self, other):
         return OrValidator(self, other)
 
+    def __and__(self, other):
+        return AndValidator(self, other)
+
 
 class OrValidator(Validator):
     def __init__(self, lhs: Validator, rhs: Validator):
         super().__init__(lambda x: lhs(x) or rhs(x))
 
 
+class AndValidator(Validator):
+    def __init__(self, lhs: Validator, rhs: Validator):
+        super().__init__(lambda x: lhs(x) and rhs(x))
+
+
 is_null = Validator(lambda x: x is None)
+is_notnull = Validator(lambda x: x is not None)
 is_bool = Validator(lambda x: isinstance(x, bool))
 is_float = Validator(lambda x: isinstance(x, float))
 is_integer = Validator(lambda x: isinstance(x, int))
@@ -69,6 +82,30 @@ def is_all_dict_keys_in(*keys):
     return Validator(lambda x: x in keys_set)
 
 
+def is_less_than(upper_bound):
+    return Validator(
+        lambda x: is_notnull(x) and is_notnull(upper_bound) and x < upper_bound
+    )
+
+
+def is_less_than_or_equal_to(upper_bound):
+    return Validator(
+        lambda x: is_notnull(x) and is_notnull(upper_bound) and x <= upper_bound
+    )
+
+
+def is_great_than(lower_bound):
+    return Validator(
+        lambda x: is_notnull(x) and is_notnull(lower_bound) and x > lower_bound
+    )
+
+
+def is_great_than_or_equal_to(lower_bound):
+    return Validator(
+        lambda x: is_notnull(x) and is_notnull(lower_bound) and x >= lower_bound
+    )
+
+
 def _is_valid_cache_path(path: str) -> bool:
     """
     path should look like oss://oss_endpoint/oss_bucket/path
@@ -91,3 +128,15 @@ _invalid_char_in_yaml_str = {'"', "'", "\n", "\\"}
 def simple_yaml_str_validator(name: str) -> bool:
     chars = set(name)
     return len(_invalid_char_in_yaml_str & chars) == 0
+
+
+def dtype_backend_validator(name: str) -> bool:
+    from ..utils import pd_release_version
+
+    check_pd_version = not str_to_bool(os.getenv(env.MAXFRAME_INSIDE_TASK))
+    name = "pyarrow" if name == "arrow" else name
+    if name not in (None, "numpy", "pyarrow"):
+        return False
+    if check_pd_version and name == "pyarrow" and pd_release_version[:2] < (1, 5):
+        raise ValueError("Need pandas>=1.5 to use pyarrow backend")
+    return True

maxframe/conftest.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import contextlib
 import faulthandler
 import os
 from configparser import ConfigParser, NoOptionError, NoSectionError
@@ -60,6 +61,10 @@ def _get_account_env(test_config: ConfigParser, section_name: str) -> ODPS:
         tunnel_endpoint = test_config.get("odps", "tunnel_endpoint")
     except NoOptionError:
         tunnel_endpoint = None
+    try:
+        namespace = test_config.get("odps", "namespace")
+    except NoOptionError:
+        namespace = None
     return ODPS(
         access_id,
         secret_access_key,
@@ -67,6 +72,7 @@ def _get_account_env(test_config: ConfigParser, section_name: str) -> ODPS:
         endpoint,
         tunnel_endpoint=tunnel_endpoint,
         overwrite_global=False,
+        namespace=namespace,
     )
 
 
@@ -84,24 +90,30 @@ def _get_bearer_token_env(test_config: ConfigParser, section_name: str) -> ODPS:
         project=entry.project,
         endpoint=entry.endpoint,
         tunnel_endpoint=entry.tunnel_endpoint,
+        namespace=entry.namespace,
     )
 
 
-@pytest.fixture(scope="session")
-def odps_with_schema(test_config):
-    try:
-        return _get_bearer_token_env(test_config, "odps_with_schema")
-    except NoSectionError:
-        pytest.skip("Need to specify odps_with_schema section in test.conf")
-
-
-@pytest.fixture(scope="session", autouse=True)
-def odps_envs(test_config):
-    entry = _get_bearer_token_env(test_config, "odps")
+@contextlib.contextmanager
+def _enter_odps_envs(entry, drop_temp_tables=True):
+    stored_envs = {}
+    for env_name in (
+        "ODPS_BEARER_TOKEN",
+        "ODPS_PROJECT_NAME",
+        "ODPS_ENDPOINT",
+        "RAY_ISOLATION_UT_ENV",
+        "ODPS_TUNNEL_ENDPOINT",
+        "ODPS_NAMESPACE",
+    ):
+        if env_name in os.environ:
+            stored_envs[env_name] = os.environ[env_name]
+            del os.environ[env_name]
 
     os.environ["ODPS_BEARER_TOKEN"] = entry.account.token
     os.environ["ODPS_PROJECT_NAME"] = entry.project
     os.environ["ODPS_ENDPOINT"] = entry.endpoint
+    if entry.namespace:
+        os.environ["ODPS_NAMESPACE"] = entry.namespace
     os.environ["RAY_ISOLATION_UT_ENV"] = "UT"
     if entry.tunnel_endpoint:
         os.environ["ODPS_TUNNEL_ENDPOINT"] = entry.tunnel_endpoint
@@ -112,16 +124,41 @@ def odps_envs(test_config):
         os.environ.pop("ODPS_BEARER_TOKEN", None)
         os.environ.pop("ODPS_PROJECT_NAME", None)
         os.environ.pop("ODPS_ENDPOINT", None)
+        os.environ.pop("ODPS_NAMESPACE", None)
         os.environ.pop("ODPS_TUNNEL_ENDPOINT", None)
         os.environ.pop("RAY_ISOLATION_UT_ENV", None)
 
-        from .tests.utils import _test_tables_to_drop
+        for env_name, val in stored_envs.items():
+            os.environ[env_name] = val
+
+        if drop_temp_tables:
+            from .tests.utils import _test_tables_to_drop
+
+            for table_name in _test_tables_to_drop:
+                try:
+                    entry.delete_table(table_name, wait=False)
+                except:
+                    pass
 
-        for table_name in _test_tables_to_drop:
-            try:
-                entry.delete_table(table_name, wait=False)
-            except:
-                pass
+
+@pytest.fixture
+def odps_with_schema(test_config, request):
+    try:
+        entry = _get_bearer_token_env(test_config, "odps_with_schema")
+    except NoSectionError:
+        pytest.skip("Need to specify odps_with_schema section in test.conf")
+        raise
+
+    with _enter_odps_envs(entry, drop_temp_tables=False):
+        yield entry
+
+
+@pytest.fixture(scope="session", autouse=True)
+def odps_envs(test_config):
+    entry = _get_bearer_token_env(test_config, "odps")
+
+    with _enter_odps_envs(entry):
+        yield
 
 
 @pytest.fixture(scope="session")

maxframe/core/accessor.py

@@ -20,13 +20,13 @@ class BaseMaxFrameAccessor:
         self.obj = obj
 
     @classmethod
-    def _register(cls, name, func):
+    def _register(cls, name, func, is_property=False):
         @functools.wraps(func)
         def wrapped(self, *args, **kw):
             return func(self.obj, *args, **kw)
 
         wrapped.__name__ = name
-        setattr(cls, name, wrapped)
+        setattr(cls, name, wrapped if not is_property else property(wrapped))
         if hasattr(cls, "_api_count"):  # pragma: no branch
             cls._api_count += 1
 

maxframe/core/base.py

@@ -126,12 +126,13 @@ class Base(Serializable):
 
     def to_kv(
         self,
-        exclude_fields: Tuple[str],
+        exclude_fields: Tuple[str] = None,
         accept_value_types: Optional[Tuple[Type]] = None,
     ):
         fields = self._FIELDS
         kv = {}
         no_value = object()
+        exclude_fields = exclude_fields or ()
         for name, field in fields.items():
             if name not in exclude_fields:
                 value = getattr(self, name, no_value)

maxframe/core/entity/core.py

@@ -53,6 +53,11 @@ class EntityData(Base):
     def is_sparse(self):
         return self.op.is_sparse()
 
+    def is_fetch(self):
+        from ..operator import Fetch
+
+        return isinstance(self.op, Fetch)
+
     issparse = is_sparse
 
     @property

maxframe/core/entity/tileables.py

@@ -312,12 +312,14 @@ class HasShapeTileableData(TileableData):
 
     @property
     def ndim(self):
-        return len(self.shape)
+        return len(self.shape) if self.shape is not None else np.nan
 
     def __len__(self):
         try:
             return int(self.shape[0])
         except (IndexError, ValueError):  # pragma: no cover
+            if len(self.shape) == 0:
+                raise TypeError("len() of unsized object") from None
             return 0
 
     @property