maxframe 2.4.0rc1__cp312-cp312-win32.whl

maxframe/config/tests/test_config.py

@@ -0,0 +1,114 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import threading
+
+import pytest
+
+from ...utils import update_wlm_quota_settings
+from ..config import Config, is_integer, is_string, option_context, options
+
+
+def test_config_context():
+    with pytest.raises(AttributeError):
+        _ = options.a.b.c
+
+    options.register_option("c.d.e", "a", is_string)
+    assert "c" in dir(options)
+    assert "d" in dir(options.c)
+
+    try:
+        with option_context() as ctx:
+            ctx.register_option("a.b.c", 1, validator=is_integer)
+            assert ctx.a.b.c == 1
+
+            ctx.a.b.c = 2
+            assert ctx.a.b.c == 2
+
+            with pytest.raises(ValueError):
+                ctx.a.b.c = "a"
+
+            assert ctx.c.d.e == "a"
+
+            ctx.c.d.e = "b"
+
+        assert options.c.d.e == "a"
+
+        options.c.d.e = "c"
+
+        assert options.c.d.e == "c"
+
+        with pytest.raises(AttributeError):
+            _ = options.a.b.c  # noqa: F841
+    finally:
+        options.unregister_option("c.d.e")
+
+
+def test_multi_thread_config():
+    options.register_option("a.b.c", 1)
+
+    class T(threading.Thread):
+        def __init__(self, is_first, condition):
+            super().__init__()
+            self.is_first = is_first
+            self.condition = condition
+
+        def run(self):
+            self.condition.acquire()
+            if self.is_first:
+                options.a.b.c = 2
+                self.condition.notify()
+            else:
+                self.condition.wait()
+                assert options.a.b.c == 2
+            self.condition.release()
+
+    try:
+        cond = threading.Condition()
+        a = T(True, cond)
+        b = T(False, cond)
+        b.start()
+        a.start()
+        a.join()
+        b.join()
+    finally:
+        options.unregister_option("a.b.c")
+
+
+def test_config_copy():
+    cfg = Config()
+    cfg.register_option("a.b.c", 1)
+    cfg.redirect_option("a.c", "a.b.c")
+
+    target_cfg = Config()
+    target_cfg.register_option("a.b.c", -1)
+    target_cfg.redirect_option("a.c", "a.b.c")
+
+    src_cfg_dict = cfg.to_dict()
+    assert src_cfg_dict == {"a.b.c": 1}
+
+    target_cfg.update(src_cfg_dict)
+    assert target_cfg.a.b.c == 1
+
+
+def test_update_wlm_quota_settings():
+    with option_context({}):
+        options.session.quota_name = "quota1"
+        engine_settings = {}
+        update_wlm_quota_settings("session_id", engine_settings)
+        assert engine_settings["odps.task.wlm.quota"] == "quota1"
+        options.session.quota_name = None
+        with pytest.raises(ValueError):
+            update_wlm_quota_settings("session_id", engine_settings)
+        assert "odps.task.wlm.quota" in engine_settings

maxframe/config/tests/test_validators.py

@@ -0,0 +1,46 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+from ..validators import (
+    is_less_than_or_equal_to,
+    is_positive_integer,
+    simple_yaml_str_validator,
+)
+
+
+@pytest.mark.parametrize("value", ["a", "http://127.0.0.1:1234", "a-b#", "ab_", "123"])
+def test_simple_yaml_str_validator_valid(value):
+    assert simple_yaml_str_validator(value)
+
+
+@pytest.mark.parametrize("value", ['a"', "'hacked'", "ab\n", "abc\\"])
+def test_simple_yaml_str_validator_invalid(value):
+    assert not simple_yaml_str_validator(value)
+
+
+@pytest.mark.parametrize(
+    "value,valid", [("a", False), (1, True), (0, False), (-1, False)]
+)
+def test_is_positive_integer_validator(value, valid):
+    assert is_positive_integer(value) is valid
+
+
+@pytest.mark.parametrize(
+    "value,upper_bound,valid",
+    [(3, 5, True), (5, 5, True), (6, 5, False), (None, None, False), (None, 5, False)],
+)
+def test_is_less_than_or_equal_to_validator(value, upper_bound, valid):
+    assert is_less_than_or_equal_to(upper_bound)(value) is valid

maxframe/config/validators.py

@@ -0,0 +1,142 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from typing import Callable
+from urllib.parse import urlparse
+
+from .. import env
+from ..utils import str_to_bool
+
+ValidatorType = Callable[..., bool]
+
+
+def any_validator(*validators: ValidatorType):
+    def validate(x):
+        return any(validator(x) for validator in validators)
+
+    return validate
+
+
+def all_validator(*validators: ValidatorType):
+    def validate(x):
+        return all(validator(x) for validator in validators)
+
+    validate.validators = validators
+    return validate
+
+
+class Validator:
+    def __init__(self, func: ValidatorType):
+        self._func = func
+
+    def __call__(self, arg) -> bool:
+        return self._func(arg)
+
+    def __or__(self, other):
+        return OrValidator(self, other)
+
+    def __and__(self, other):
+        return AndValidator(self, other)
+
+
+class OrValidator(Validator):
+    def __init__(self, lhs: Validator, rhs: Validator):
+        super().__init__(lambda x: lhs(x) or rhs(x))
+
+
+class AndValidator(Validator):
+    def __init__(self, lhs: Validator, rhs: Validator):
+        super().__init__(lambda x: lhs(x) and rhs(x))
+
+
+is_null = Validator(lambda x: x is None)
+is_notnull = Validator(lambda x: x is not None)
+is_bool = Validator(lambda x: isinstance(x, bool))
+is_float = Validator(lambda x: isinstance(x, float))
+is_integer = Validator(lambda x: isinstance(x, int))
+is_numeric = Validator(lambda x: isinstance(x, (int, float)))
+is_string = Validator(lambda x: isinstance(x, str))
+is_dict = Validator(lambda x: isinstance(x, dict))
+is_positive_integer = Validator(lambda x: is_integer(x) and x > 0)
+is_non_negative_integer = Validator(lambda x: is_integer(x) and x >= 0)
+
+
+def is_in(vals):
+    return Validator(vals.__contains__)
+
+
+def is_all_dict_keys_in(*keys):
+    keys_set = set(keys)
+    return Validator(lambda x: x in keys_set)
+
+
+def is_less_than(upper_bound):
+    return Validator(
+        lambda x: is_notnull(x) and is_notnull(upper_bound) and x < upper_bound
+    )
+
+
+def is_less_than_or_equal_to(upper_bound):
+    return Validator(
+        lambda x: is_notnull(x) and is_notnull(upper_bound) and x <= upper_bound
+    )
+
+
+def is_great_than(lower_bound):
+    return Validator(
+        lambda x: is_notnull(x) and is_notnull(lower_bound) and x > lower_bound
+    )
+
+
+def is_great_than_or_equal_to(lower_bound):
+    return Validator(
+        lambda x: is_notnull(x) and is_notnull(lower_bound) and x >= lower_bound
+    )
+
+
+def _is_valid_cache_path(path: str) -> bool:
+    """
+    path should look like oss://oss_endpoint/oss_bucket/path
+    """
+    parsed_url = urlparse(path)
+    return (
+        parsed_url.scheme == "oss"
+        and parsed_url.netloc
+        and parsed_url.path
+        and "/" in parsed_url.path
+    )
+
+
+is_valid_cache_path = Validator(_is_valid_cache_path)
+
+
+_invalid_char_in_yaml_str = {'"', "'", "\n", "\\"}
+
+
+def simple_yaml_str_validator(name: str) -> bool:
+    chars = set(name)
+    return len(_invalid_char_in_yaml_str & chars) == 0
+
+
+def dtype_backend_validator(name: str) -> bool:
+    from ..utils import pd_release_version
+
+    check_pd_version = not str_to_bool(os.getenv(env.MAXFRAME_INSIDE_TASK))
+    name = "pyarrow" if name == "arrow" else name
+    if name not in (None, "numpy", "pyarrow"):
+        return False
+    if check_pd_version and name == "pyarrow" and pd_release_version[:2] < (1, 5):
+        raise ValueError("Need pandas>=1.5 to use pyarrow backend")
+    return True

maxframe/conftest.py

@@ -0,0 +1,261 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import contextlib
+import faulthandler
+import os
+from configparser import ConfigParser, NoOptionError, NoSectionError
+
+import pytest
+from odps import ODPS
+from odps.accounts import BearerTokenAccount
+
+from .config import options
+
+faulthandler.enable(all_threads=True)
+_test_conf_file_name = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)), "tests", "test.conf"
+)
+
+
+@pytest.fixture(scope="session")
+def test_config():
+    config = ConfigParser()
+    config.read(_test_conf_file_name)
+    return config
+
+
+def _get_account_env(test_config: ConfigParser, section_name: str) -> ODPS:
+    try:
+        access_id = test_config.get(section_name, "access_id")
+    except NoOptionError:
+        access_id = test_config.get("odps", "access_id")
+    if not access_id:
+        access_id = os.getenv("ACCESS_ID")
+    try:
+        secret_access_key = test_config.get(section_name, "secret_access_key")
+    except NoOptionError:
+        secret_access_key = test_config.get("odps", "secret_access_key")
+    if not secret_access_key:
+        secret_access_key = os.getenv("SECRET_ACCESS_KEY")
+    try:
+        project = test_config.get(section_name, "project")
+    except NoOptionError:
+        project = test_config.get("odps", "project")
+    try:
+        endpoint = test_config.get(section_name, "endpoint")
+    except NoOptionError:
+        endpoint = test_config.get("odps", "endpoint")
+    try:
+        tunnel_endpoint = test_config.get("odps", "tunnel_endpoint")
+    except NoOptionError:
+        tunnel_endpoint = None
+    try:
+        namespace = test_config.get("odps", "namespace")
+    except NoOptionError:
+        namespace = None
+    return ODPS(
+        access_id,
+        secret_access_key,
+        project,
+        endpoint,
+        tunnel_endpoint=tunnel_endpoint,
+        overwrite_global=False,
+        namespace=namespace,
+    )
+
+
+def _get_bearer_token_env(test_config: ConfigParser, section_name: str) -> ODPS:
+    entry = _get_account_env(test_config, section_name)
+    policy = {
+        "Version": "1",
+        "Statement": [
+            {"Action": ["odps:*"], "Resource": "acs:odps:*:*", "Effect": "Allow"}
+        ],
+    }
+    token = entry.get_project().generate_auth_token(policy, "bearer", 5)
+    return ODPS(
+        account=BearerTokenAccount(token, 5),
+        project=entry.project,
+        endpoint=entry.endpoint,
+        tunnel_endpoint=entry.tunnel_endpoint,
+        namespace=entry.namespace,
+    )
+
+
+@contextlib.contextmanager
+def _enter_odps_envs(entry, drop_temp_tables=True):
+    stored_envs = {}
+    for env_name in (
+        "ODPS_BEARER_TOKEN",
+        "ODPS_PROJECT_NAME",
+        "ODPS_ENDPOINT",
+        "RAY_ISOLATION_UT_ENV",
+        "ODPS_TUNNEL_ENDPOINT",
+        "ODPS_NAMESPACE",
+    ):
+        if env_name in os.environ:
+            stored_envs[env_name] = os.environ[env_name]
+            del os.environ[env_name]
+
+    os.environ["ODPS_BEARER_TOKEN"] = entry.account.token
+    os.environ["ODPS_PROJECT_NAME"] = entry.project
+    os.environ["ODPS_ENDPOINT"] = entry.endpoint
+    if entry.namespace:
+        os.environ["ODPS_NAMESPACE"] = entry.namespace
+    os.environ["RAY_ISOLATION_UT_ENV"] = "UT"
+    if entry.tunnel_endpoint:
+        os.environ["ODPS_TUNNEL_ENDPOINT"] = entry.tunnel_endpoint
+
+    try:
+        yield
+    finally:
+        os.environ.pop("ODPS_BEARER_TOKEN", None)
+        os.environ.pop("ODPS_PROJECT_NAME", None)
+        os.environ.pop("ODPS_ENDPOINT", None)
+        os.environ.pop("ODPS_NAMESPACE", None)
+        os.environ.pop("ODPS_TUNNEL_ENDPOINT", None)
+        os.environ.pop("RAY_ISOLATION_UT_ENV", None)
+
+        for env_name, val in stored_envs.items():
+            os.environ[env_name] = val
+
+        if drop_temp_tables:
+            from .tests.utils import _test_tables_to_drop
+
+            for table_name in _test_tables_to_drop:
+                try:
+                    entry.delete_table(table_name, wait=False)
+                except:
+                    pass
+
+
+@pytest.fixture
+def odps_with_schema(test_config, request):
+    try:
+        entry = _get_bearer_token_env(test_config, "odps_with_schema")
+    except NoSectionError:
+        pytest.skip("Need to specify odps_with_schema section in test.conf")
+        raise
+
+    with _enter_odps_envs(entry, drop_temp_tables=False):
+        yield entry
+
+
+@pytest.fixture(scope="session", autouse=True)
+def odps_envs(test_config):
+    entry = _get_bearer_token_env(test_config, "odps")
+
+    with _enter_odps_envs(entry):
+        yield
+
+
+@pytest.fixture(scope="session")
+def odps_account(test_config):
+    return _get_account_env(test_config, "odps")
+
+
+@pytest.fixture(scope="session")
+def oss_config():
+    config = ConfigParser()
+    config.read(_test_conf_file_name)
+
+    old_role_arn = options.service_role_arn
+    old_cache_url = options.object_cache_url
+
+    try:
+        oss_access_id = config.get("oss", "access_id") or os.getenv("ACCESS_ID")
+        oss_secret_access_key = config.get("oss", "secret_access_key") or os.getenv(
+            "SECRET_ACCESS_KEY"
+        )
+        oss_bucket_name = config.get("oss", "bucket_name")
+        oss_endpoint = config.get("oss", "endpoint")
+        oss_rolearn = config.get("oss", "rolearn")
+
+        options.service_role_arn = oss_rolearn
+        if "test" in oss_endpoint:
+            oss_svc_endpoint = oss_endpoint
+        else:
+            endpoint_parts = oss_endpoint.split(".", 1)
+            if "-internal" not in endpoint_parts[0]:
+                endpoint_parts[0] += "-internal"
+            oss_svc_endpoint = ".".join(endpoint_parts)
+        options.object_cache_url = f"oss://{oss_svc_endpoint}/{oss_bucket_name}"
+
+        config.oss_config = (
+            oss_access_id,
+            oss_secret_access_key,
+            oss_bucket_name,
+            oss_endpoint,
+        )
+
+        import oss2
+
+        auth = oss2.Auth(oss_access_id, oss_secret_access_key)
+        config.oss_bucket = oss2.Bucket(auth, oss_endpoint, oss_bucket_name)
+        config.oss_rolearn = oss_rolearn
+        yield config
+    except (NoSectionError, NoOptionError, ImportError):
+        yield None
+    finally:
+        options.service_role_arn = old_role_arn
+        options.object_cache_url = old_cache_url
+
+
+@pytest.fixture(autouse=True)
+def apply_engine_selection(request):
+    try:
+        from maxframe_framedriver.services.analyzer import DagAnalyzer
+    except ImportError:
+        DagAnalyzer = None
+    try:
+        marks = list(request.node.iter_markers())
+        if request.node.parent:
+            marks.extend(request.node.parent.iter_markers())
+        marks = [m for m in marks if m.name == "maxframe_engine"]
+
+        if DagAnalyzer:
+            engines = set()
+            for mark in marks:
+                engines.update(mark.args[0])
+            DagAnalyzer._enabled_engines = set(engines)
+        yield
+    finally:
+        if DagAnalyzer:
+            DagAnalyzer._enabled_engines = set()
+
+
+@pytest.fixture
+def local_test_envs():
+    spe_launcher_env = "MAXFRAME_SPE_LAUNCHER"
+    old_value = os.getenv(spe_launcher_env)
+    os.environ[spe_launcher_env] = "local"
+    yield
+    if old_value is not None:
+        os.environ[spe_launcher_env] = old_value
+    else:
+        del os.environ[spe_launcher_env]
+
+
+@pytest.fixture
+def enable_local_execution(request):
+    old_enabled = options.local_execution.enabled
+    old_limit = options.local_execution.size_limit
+    try:
+        options.local_execution.enabled = True
+        options.local_execution.size_limit = getattr(request, "param", 0)
+        yield
+    finally:
+        options.local_execution.enabled = old_enabled
+        options.local_execution.size_limit = old_limit

maxframe/core/__init__.py

@@ -0,0 +1,53 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# noinspection PyUnresolvedReferences
+from ..typing_ import ChunkType, EntityType, OperatorType, TileableType
+from .accessor import BaseMaxFrameAccessor, CachedAccessor
+from .base import Base, ExecutionError
+from .entity import (
+    ENTITY_TYPE,
+    OBJECT_TYPE,
+    TILEABLE_TYPE,
+    Entity,
+    EntityData,
+    ExecutableTuple,
+    HasShapeTileable,
+    HasShapeTileableData,
+    NotSupportTile,
+    Object,
+    ObjectData,
+    OutputType,
+    Tileable,
+    TileableData,
+    _ExecuteAndFetchMixin,
+    df_output_types,
+    get_fetch_class,
+    get_output_types,
+    get_tileable_types,
+    register_fetch_class,
+    register_output_types,
+)
+
+# noinspection PyUnresolvedReferences
+from .graph import (
+    DAG,
+    DirectedGraph,
+    GraphContainsCycleError,
+    GraphSerializer,
+    TileableGraph,
+    TileableGraphBuilder,
+)
+from .mode import enter_mode, is_build_mode, is_eager_mode, is_kernel_mode
+from .operator import build_fetch

maxframe/core/accessor.py

@@ -0,0 +1,45 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import functools
+
+
+class BaseMaxFrameAccessor:
+    def __init__(self, obj):
+        self.obj = obj
+
+    @classmethod
+    def _register(cls, name, func, is_property=False):
+        @functools.wraps(func)
+        def wrapped(self, *args, **kw):
+            return func(self.obj, *args, **kw)
+
+        wrapped.__name__ = name
+        setattr(cls, name, wrapped if not is_property else property(wrapped))
+        if hasattr(cls, "_api_count"):  # pragma: no branch
+            cls._api_count += 1
+
+
+class CachedAccessor:
+    def __init__(self, name: str, accessor) -> None:
+        self._name = name
+        self._accessor = accessor
+
+    def __get__(self, obj, cls):
+        if obj is None:
+            # we're accessing the attribute of the class, i.e., Dataset.geo
+            return self._accessor
+        if self._name not in obj._accessors:
+            obj._accessors[self._name] = self._accessor(obj)
+        return obj._accessors[self._name]