maxframe 1.0.0rc2__cp38-cp38-macosx_10_9_universal2.whl → 1.0.0rc4__cp38-cp38-macosx_10_9_universal2.whl

maxframe/codegen.py

@@ -26,9 +26,9 @@ from odps.utils import camel_to_underline
 from .core import OperatorType, Tileable, TileableGraph
 from .core.operator import Fetch
 from .extension import iter_extensions
+from .io.odpsio import build_dataframe_table_meta
+from .io.odpsio.schema import pandas_to_odps_schema
 from .lib import wrapped_pickle as pickle
-from .odpsio import build_dataframe_table_meta
-from .odpsio.schema import pandas_to_odps_schema
 from .protocol import DataFrameTableMeta, ResultInfo
 from .serialization import PickleContainer
 from .serialization.serializables import Serializable, StringField
@@ -347,6 +347,7 @@ BUILTIN_ENGINE_SPE = "SPE"
 BUILTIN_ENGINE_MCSQL = "MCSQL"
 
 FAST_RANGE_INDEX_ENABLED = "codegen.fast_range_index_enabled"
+ROW_NUMBER_WINDOW_INDEX_ENABLED = "codegen.row_number_window_index_enabled"
 
 
 class BigDagCodeGenerator(metaclass=abc.ABCMeta):
@@ -506,6 +507,7 @@ class BigDagCodeGenerator(metaclass=abc.ABCMeta):
                 prefer_binary=pack.prefer_binary,
                 pre_release=pack.pre_release,
                 force_rebuild=pack.force_rebuild,
+                no_audit_wheel=pack.no_audit_wheel,
                 python_tag=python_tag,
                 is_production=is_production,
                 schedule_id=schedule_id,

maxframe/config/config.py

@@ -32,7 +32,6 @@ from ..utils import get_python_tag
 from .validators import (
     ValidatorType,
     all_validator,
-    any_validator,
     is_bool,
     is_dict,
     is_in,
@@ -41,6 +40,7 @@ from .validators import (
     is_null,
     is_numeric,
     is_string,
+    is_valid_cache_path,
 )
 
 _DEFAULT_REDIRECT_WARN = "Option {source} has been replaced by {target} and might be removed in a future release."
@@ -327,7 +327,7 @@ default_options.register_option(
 default_options.register_option(
     "local_timezone",
     _get_legal_local_tz_name(),
-    validator=any_validator(is_null, is_in(set(available_timezones()))),
+    validator=is_null | is_in(set(available_timezones())),
     remote=True,
 )
 default_options.register_option(
@@ -343,11 +343,25 @@ default_options.register_option("sql.enable_mcqa", True, validator=is_bool, remo
 default_options.register_option(
     "sql.generate_comments", True, validator=is_bool, remote=True
 )
+default_options.register_option(
+    "sql.auto_use_common_image", True, validator=is_bool, remote=True
+)
 default_options.register_option("sql.settings", {}, validator=is_dict, remote=True)
 
 default_options.register_option("is_production", False, validator=is_bool, remote=True)
 default_options.register_option("schedule_id", "", validator=is_string, remote=True)
 
+default_options.register_option(
+    "service_role_arn", None, validator=is_null | is_string, remote=True
+)
+default_options.register_option(
+    "object_cache_url", None, validator=is_null | is_valid_cache_path, remote=True
+)
+
+default_options.register_option(
+    "chunk_size", None, validator=is_null | is_integer, remote=True
+)
+
 default_options.register_option(
     "session.max_alive_seconds",
     _DEFAULT_MAX_ALIVE_SECONDS,
@@ -360,15 +374,22 @@ default_options.register_option(
     validator=is_numeric,
     remote=True,
 )
+default_options.register_option(
+    "session.quota_name", None, validator=is_null | is_string, remote=True
+)
+default_options.register_option(
+    "session.enable_schema", None, validator=is_null | is_bool, remote=True
+)
+default_options.register_option(
+    "session.default_schema", None, validator=is_null | is_string, remote=True
+)
 default_options.register_option(
     "session.upload_batch_size",
     _DEFAULT_UPLOAD_BATCH_SIZE,
     validator=is_integer,
 )
 default_options.register_option(
-    "session.table_lifecycle",
-    None,
-    validator=any_validator(is_null, is_integer),
+    "session.table_lifecycle", None, validator=is_null | is_integer, remote=True
 )
 default_options.register_option(
     "session.temp_table_lifecycle",
@@ -379,7 +400,7 @@ default_options.register_option(
 default_options.register_option(
     "session.subinstance_priority",
     None,
-    validator=any_validator(is_null, is_integer),
+    validator=is_null | is_integer,
     remote=True,
 )
 
@@ -391,9 +412,7 @@ default_options.register_option(
 default_options.register_option(
     "optimize.head_optimize_threshold", 1000, validator=is_integer
 )
-default_options.register_option(
-    "show_progress", "auto", validator=any_validator(is_bool, is_string)
-)
+default_options.register_option("show_progress", "auto", validator=is_bool | is_string)
 default_options.register_option(
     "dag.settings", value=dict(), validator=is_dict, remote=True
 )

maxframe/config/validators.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 from typing import Callable
+from urllib.parse import urlparse
 
 ValidatorType = Callable[..., bool]
 
@@ -32,22 +33,51 @@ def all_validator(*validators: ValidatorType):
     return validate
 
 
-is_null = lambda x: x is None
-is_bool = lambda x: isinstance(x, bool)
-is_float = lambda x: isinstance(x, float)
-is_integer = lambda x: isinstance(x, int)
-is_numeric = lambda x: isinstance(x, (int, float))
-is_string = lambda x: isinstance(x, str)
-is_dict = lambda x: isinstance(x, dict)
-is_positive_integer = lambda x: is_integer(x) and x > 0
-is_non_negative_integer = lambda x: is_integer(x) and x >= 0
+class Validator:
+    def __init__(self, func: ValidatorType):
+        self._func = func
+
+    def __call__(self, arg) -> bool:
+        return self._func(arg)
+
+    def __or__(self, other):
+        return OrValidator(self, other)
+
+
+class OrValidator(Validator):
+    def __init__(self, lhs: Validator, rhs: Validator):
+        super().__init__(lambda x: lhs(x) or rhs(x))
+
+
+is_null = Validator(lambda x: x is None)
+is_bool = Validator(lambda x: isinstance(x, bool))
+is_float = Validator(lambda x: isinstance(x, float))
+is_integer = Validator(lambda x: isinstance(x, int))
+is_numeric = Validator(lambda x: isinstance(x, (int, float)))
+is_string = Validator(lambda x: isinstance(x, str))
+is_dict = Validator(lambda x: isinstance(x, dict))
+is_positive_integer = Validator(lambda x: is_integer(x) and x > 0)
+is_non_negative_integer = Validator(lambda x: is_integer(x) and x >= 0)
 
 
 def is_in(vals):
-    def validate(x):
-        return x in vals
+    return Validator(vals.__contains__)
 
-    return validate
+
+def _is_valid_cache_path(path: str) -> bool:
+    """
+    path should look like oss://oss_endpoint/oss_bucket/path
+    """
+    parsed_url = urlparse(path)
+    return (
+        parsed_url.scheme == "oss"
+        and parsed_url.netloc
+        and parsed_url.path
+        and "/" in parsed_url.path
+    )
+
+
+is_valid_cache_path = Validator(_is_valid_cache_path)
 
 
 _invalid_char_in_yaml_str = {'"', "'", "\n", "\\"}

maxframe/conftest.py

@@ -14,10 +14,13 @@
 
 import faulthandler
 import os
-from configparser import ConfigParser, NoOptionError
+from configparser import ConfigParser, NoOptionError, NoSectionError
 
 import pytest
 from odps import ODPS
+from odps.accounts import BearerTokenAccount
+
+from .config import options
 
 faulthandler.enable(all_threads=True)
 _test_conf_file_name = os.path.join(
@@ -32,12 +35,23 @@ def test_config():
     return config
 
 
-@pytest.fixture(scope="session", autouse=True)
-def odps_envs(test_config):
-    access_id = test_config.get("odps", "access_id")
-    secret_access_key = test_config.get("odps", "secret_access_key")
-    project = test_config.get("odps", "project")
-    endpoint = test_config.get("odps", "endpoint")
+def _get_odps_env(test_config: ConfigParser, section_name: str) -> ODPS:
+    try:
+        access_id = test_config.get(section_name, "access_id")
+    except NoOptionError:
+        access_id = test_config.get("odps", "access_id")
+    try:
+        secret_access_key = test_config.get(section_name, "secret_access_key")
+    except NoOptionError:
+        secret_access_key = test_config.get("odps", "secret_access_key")
+    try:
+        project = test_config.get(section_name, "project")
+    except NoOptionError:
+        project = test_config.get("odps", "project")
+    try:
+        endpoint = test_config.get(section_name, "endpoint")
+    except NoOptionError:
+        endpoint = test_config.get("odps", "endpoint")
     try:
         tunnel_endpoint = test_config.get("odps", "tunnel_endpoint")
     except NoOptionError:
@@ -53,12 +67,31 @@ def odps_envs(test_config):
         ],
     }
     token = entry.get_project().generate_auth_token(policy, "bearer", 5)
+    return ODPS(
+        account=BearerTokenAccount(token, 5),
+        project=project,
+        endpoint=endpoint,
+        tunnel_endpoint=tunnel_endpoint,
+    )
 
-    os.environ["ODPS_BEARER_TOKEN"] = token
-    os.environ["ODPS_PROJECT_NAME"] = project
-    os.environ["ODPS_ENDPOINT"] = endpoint
-    if tunnel_endpoint:
-        os.environ["ODPS_TUNNEL_ENDPOINT"] = tunnel_endpoint
+
+@pytest.fixture(scope="session")
+def odps_with_schema(test_config):
+    try:
+        return _get_odps_env(test_config, "odps_with_schema")
+    except NoSectionError:
+        pytest.skip("Need to specify odps_with_schema section in test.conf")
+
+
+@pytest.fixture(scope="session", autouse=True)
+def odps_envs(test_config):
+    entry = _get_odps_env(test_config, "odps")
+
+    os.environ["ODPS_BEARER_TOKEN"] = entry.account.token
+    os.environ["ODPS_PROJECT_NAME"] = entry.project
+    os.environ["ODPS_ENDPOINT"] = entry.endpoint
+    if entry.tunnel_endpoint:
+        os.environ["ODPS_TUNNEL_ENDPOINT"] = entry.tunnel_endpoint
 
     try:
         yield
@@ -77,11 +110,14 @@ def odps_envs(test_config):
                 pass
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
 def oss_config():
     config = ConfigParser()
     config.read(_test_conf_file_name)
 
+    old_role_arn = options.service_role_arn
+    old_cache_url = options.object_cache_url
+
     try:
         oss_access_id = config.get("oss", "access_id")
         oss_secret_access_key = config.get("oss", "secret_access_key")
@@ -89,6 +125,9 @@ def oss_config():
         oss_endpoint = config.get("oss", "endpoint")
         oss_rolearn = config.get("oss", "rolearn")
 
+        options.service_role_arn = oss_rolearn
+        options.object_cache_url = f"oss://{oss_endpoint}/{oss_bucket_name}"
+
         config.oss_config = (
             oss_access_id,
             oss_secret_access_key,
@@ -101,9 +140,12 @@ def oss_config():
         auth = oss2.Auth(oss_access_id, oss_secret_access_key)
         config.oss_bucket = oss2.Bucket(auth, oss_endpoint, oss_bucket_name)
         config.oss_rolearn = oss_rolearn
-        return config
+        yield config
     except (ConfigParser.NoSectionError, ConfigParser.NoOptionError, ImportError):
         return None
+    finally:
+        options.service_role_arn = old_role_arn
+        options.object_cache_url = old_cache_url
 
 
 @pytest.fixture(autouse=True)

maxframe/core/__init__.py

@@ -14,20 +14,14 @@
 
 # noinspection PyUnresolvedReferences
 from ..typing_ import ChunkType, EntityType, OperatorType, TileableType
-from .base import ExecutionError
+from .base import Base, ExecutionError
 from .entity import (
-    CHUNK_TYPE,
     ENTITY_TYPE,
-    FUSE_CHUNK_TYPE,
     OBJECT_TYPE,
     TILEABLE_TYPE,
-    Chunk,
-    ChunkData,
     Entity,
     EntityData,
     ExecutableTuple,
-    FuseChunk,
-    FuseChunkData,
     HasShapeTileable,
     HasShapeTileableData,
     NotSupportTile,
@@ -40,23 +34,18 @@ from .entity import (
     get_fetch_class,
     get_output_types,
     get_tileable_types,
-    register,
     register_fetch_class,
     register_output_types,
-    unregister,
 )
 
 # noinspection PyUnresolvedReferences
 from .graph import (
     DAG,
-    ChunkGraph,
-    ChunkGraphBuilder,
     DirectedGraph,
     GraphContainsCycleError,
     GraphSerializer,
     TileableGraph,
     TileableGraphBuilder,
-    TileContext,
-    TileStatus,
 )
 from .mode import enter_mode, is_build_mode, is_eager_mode, is_kernel_mode
+from .operator import build_fetch

maxframe/core/entity/__init__.py

@@ -12,10 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .chunks import CHUNK_TYPE, Chunk, ChunkData
 from .core import ENTITY_TYPE, Entity, EntityData
 from .executable import ExecutableTuple, _ExecuteAndFetchMixin
-from .fuse import FUSE_CHUNK_TYPE, FuseChunk, FuseChunkData
 from .objects import OBJECT_TYPE, Object, ObjectData
 from .output_types import (
     OutputType,
@@ -32,6 +30,4 @@ from .tileables import (
     NotSupportTile,
     Tileable,
     TileableData,
-    register,
-    unregister,
 )

maxframe/core/entity/executable.py

@@ -46,7 +46,7 @@ class DecrefRunner:
                 break
 
             session = session_ref()
-            if session is None:
+            if session is None or session.closed:
                 fut.set_result(None)
                 continue
             try:

maxframe/core/entity/objects.py

@@ -12,8 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Any, Dict
+from typing import Any, Dict, Type
 
+from ...serialization import load_type
+from ...serialization.serializables import StringField
 from .core import Entity
 from .executable import _ToObjectMixin
 from .tileables import TileableData
@@ -25,9 +27,42 @@ class ObjectData(TileableData, _ToObjectMixin):
     # workaround for removed field since v0.1.0b5
     # todo remove this when all versions below v1.0.0rc1 is eliminated
     _legacy_deprecated_non_primitives = ["_chunks"]
+    _legacy_new_non_primitives = ["object_class"]
+
+    object_class = StringField("object_class", default=None)
+
+    @classmethod
+    def get_entity_class(cls) -> Type["Object"]:
+        if getattr(cls, "_entity_class", None) is not None:
+            return cls._entity_class
+        assert cls.__qualname__[-4:] == "Data"
+        target_class_name = cls.__module__ + "#" + cls.__qualname__[:-4]
+        cls._entity_class = load_type(target_class_name, Object)
+        return cls._entity_class
+
+    def __new__(cls, op=None, nsplits=None, **kw):
+        if cls is ObjectData:
+            obj_cls = kw.get("object_class")
+            if isinstance(obj_cls, str):
+                obj_cls = load_type(obj_cls, (Object, ObjectData))
+            if isinstance(obj_cls, type) and issubclass(obj_cls, Object):
+                obj_cls = obj_cls.get_data_class()
+
+            if obj_cls is not None and cls is not obj_cls:
+                return obj_cls(op=op, nsplits=nsplits, **kw)
+        return super().__new__(cls)
 
     def __init__(self, op=None, nsplits=None, **kw):
+        obj_cls = kw.pop("object_class", None)
+        if isinstance(obj_cls, type):
+            if isinstance(obj_cls, type) and issubclass(obj_cls, Object):
+                obj_cls = obj_cls.get_data_class()
+            kw["object_class"] = obj_cls.__module__ + "#" + obj_cls.__qualname__
+
         super().__init__(_op=op, _nsplits=nsplits, **kw)
+        if self.object_class is None and type(self) is not ObjectData:
+            cls = type(self)
+            self.object_class = cls.__module__ + "#" + cls.__qualname__
 
     def __repr__(self):
         return f"Object <op={type(self.op).__name__}, key={self.key}>"
@@ -35,7 +70,7 @@ class ObjectData(TileableData, _ToObjectMixin):
     @property
     def params(self):
         # params return the properties which useful to rebuild a new tileable object
-        return dict()
+        return dict(object_class=self.object_class)
 
     @params.setter
     def params(self, new_params: Dict[str, Any]):
@@ -54,5 +89,13 @@ class Object(Entity, _ToObjectMixin):
     _allow_data_type_ = (ObjectData,)
     type_name = "Object"
 
+    @classmethod
+    def get_data_class(cls) -> Type[ObjectData]:
+        if getattr(cls, "_data_class", None) is not None:
+            return cls._data_class
+        target_class_name = cls.__module__ + "#" + cls.__qualname__ + "Data"
+        cls._data_class = load_type(target_class_name, ObjectData)
+        return cls._data_class
+
 
 OBJECT_TYPE = (Object, ObjectData)

maxframe/core/entity/output_types.py

@@ -15,7 +15,6 @@
 import functools
 from enum import Enum
 
-from .fuse import FUSE_CHUNK_TYPE
 from .objects import OBJECT_TYPE
 
 
@@ -77,8 +76,6 @@ def get_output_types(*objs, unknown_as=None):
     for obj in objs:
         if obj is None:
             continue
-        elif isinstance(obj, FUSE_CHUNK_TYPE):
-            obj = obj.chunk
 
         try:
             output_types.append(_get_output_type_by_cls(type(obj)))

maxframe/core/entity/tests/test_objects.py

@@ -0,0 +1,43 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ..objects import Object, ObjectData
+
+
+class TestSubObjectData(ObjectData):
+    __test__ = False
+
+
+class TestSubObject(Object):
+    __test__ = False
+
+
+def test_object_init():
+    assert TestSubObjectData.get_entity_class() is TestSubObject
+
+    obj = ObjectData(
+        object_class=TestSubObjectData.__module__ + "#" + TestSubObjectData.__name__
+    )
+    assert isinstance(obj, TestSubObjectData)
+
+    obj = ObjectData(object_class=TestSubObjectData)
+    assert isinstance(obj, TestSubObjectData)
+
+    obj = ObjectData(
+        object_class=TestSubObject.__module__ + "#" + TestSubObject.__name__
+    )
+    assert isinstance(obj, TestSubObjectData)
+
+    obj = ObjectData(object_class=TestSubObject)
+    assert isinstance(obj, TestSubObjectData)

maxframe/core/entity/tileables.py

@@ -15,17 +15,15 @@
 import builtins
 import itertools
 from operator import attrgetter
-from typing import Callable, List
 from weakref import WeakKeyDictionary, WeakSet
 
 import numpy as np
 
 from ...serialization.serializables import BoolField, FieldTypes, TupleField
-from ...typing_ import OperatorType, TileableType
+from ...typing_ import TileableType
 from ...utils import on_deserialize_shape, on_serialize_nsplits, on_serialize_shape
 from ..base import Base
 from ..mode import enter_mode
-from .chunks import Chunk
 from .core import Entity, EntityData
 from .executable import _ExecutableMixin
 
@@ -34,79 +32,6 @@ class NotSupportTile(Exception):
     pass
 
 
-class OperatorTilesHandler:
-    _handlers = dict()
-
-    @classmethod
-    def _get_op_cls(cls, op: OperatorType):
-        if isinstance(op, type):
-            return op
-        return type(op)
-
-    @classmethod
-    def register(
-        cls, op: OperatorType, tile_handler: Callable[[OperatorType], TileableType]
-    ):
-        cls._handlers[cls._get_op_cls(op)] = tile_handler
-
-    @classmethod
-    def unregister(cls, op: OperatorType):
-        del cls._handlers[cls._get_op_cls(op)]
-
-    @classmethod
-    def get_handler(
-        cls, op: OperatorType
-    ) -> Callable[[OperatorType], List[TileableType]]:
-        op_cls = cls._get_op_cls(op)
-        return cls._handlers.get(op_cls, op_cls.tile)
-
-    @classmethod
-    def _assign_to(
-        cls,
-        tile_after_tensor_datas: List["TileableData"],
-        tile_before_tensor_datas: List["TileableData"],
-    ):
-        assert len(tile_after_tensor_datas) == len(tile_before_tensor_datas)
-
-        for tile_after_tensor_data, tile_before_tensor_data in zip(
-            tile_after_tensor_datas, tile_before_tensor_datas
-        ):
-            if tile_before_tensor_data is None:
-                # garbage collected
-                continue
-            tile_after_tensor_data.copy_to(tile_before_tensor_data)
-            tile_before_tensor_data.op.outputs = tile_before_tensor_datas
-
-    @enter_mode(kernel=True)
-    def dispatch(self, op: OperatorType):
-        op_cls = self._get_op_cls(op)
-        tiled = None
-        cause = None
-
-        if op_cls in self._handlers:
-            tiled = self._handlers[op_cls](op)
-        else:
-            try:
-                tiled = op_cls.tile(op)
-            except NotImplementedError as ex:
-                cause = ex
-                for super_cls in op_cls.__mro__:
-                    if super_cls in self._handlers:
-                        h = self._handlers[op_cls] = self._handlers[super_cls]
-                        tiled = h(op)
-                        break
-
-        if tiled is not None:
-            return tiled if isinstance(tiled, list) else [tiled]
-        else:
-            raise NotImplementedError(f"{type(op)} does not support tile") from cause
-
-
-handler = OperatorTilesHandler()
-register = OperatorTilesHandler.register
-unregister = OperatorTilesHandler.unregister
-
-
 class _ChunksIndexer:
     __slots__ = ("_tileable",)
 
@@ -231,7 +156,7 @@ entity_view_handler = EntityDataModificationHandler()
 
 
 class TileableData(EntityData, _ExecutableMixin):
-    __slots__ = "_cix", "_entities", "_executed_sessions"
+    __slots__ = "_chunks", "_cix", "_entities", "_executed_sessions"
     _no_copy_attrs_ = Base._no_copy_attrs_ | {"_cix"}
 
     # optional fields
@@ -245,6 +170,8 @@ class TileableData(EntityData, _ExecutableMixin):
     cache = BoolField("cache", default=False)
 
     def __init__(self: TileableType, *args, **kwargs):
+        if kwargs.get("chunks") is not None:
+            self._chunks = kwargs.pop("chunks")
         if kwargs.get("_nsplits", None) is not None:
             kwargs["_nsplits"] = tuple(tuple(s) for s in kwargs["_nsplits"])
 
@@ -270,7 +197,7 @@ class TileableData(EntityData, _ExecutableMixin):
             return tuple(map(len, self._nsplits))
 
     @property
-    def chunks(self) -> List[Chunk]:
+    def chunks(self) -> list:
         return getattr(self, "_chunks", None)
 
     @property

maxframe/core/graph/__init__.py

@@ -12,6 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .builder import ChunkGraphBuilder, TileableGraphBuilder, TileContext, TileStatus
+from .builder import TileableGraphBuilder
 from .core import DAG, DirectedGraph, GraphContainsCycleError
-from .entity import ChunkGraph, EntityGraph, GraphSerializer, TileableGraph
+from .entity import EntityGraph, GraphSerializer, TileableGraph

maxframe/core/graph/builder/__init__.py

@@ -12,5 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .chunk import ChunkGraphBuilder, TileContext, TileStatus
 from .tileable import TileableGraphBuilder