maxframe 1.3.0__cp310-cp310-macosx_10_9_universal2.whl → 2.0.0__cp310-cp310-macosx_10_9_universal2.whl

maxframe/serialization/tests/test_serial.py

@@ -22,6 +22,8 @@ import numpy as np
 import pandas as pd
 import pytest
 
+from .. import PickleHookOptions
+
 try:
     import pyarrow as pa
 except ImportError:
@@ -199,6 +201,10 @@ def test_pandas():
         )
     pd.testing.assert_frame_equal(val, deserialize(*serialize(val)))
 
+    # dataframe with index only
+    val = pd.DataFrame([], index=pd.RangeIndex(1000))
+    pd.testing.assert_frame_equal(val, deserialize(*serialize(val)))
+
     val = pd.MultiIndex.from_arrays(
         [(1, 5, 4, 9, 6), list("BADCE")], names=["C1", "C2"]
     )
@@ -308,6 +314,27 @@ def test_pickle_container():
         assert deserial2() == func_to_pk()
 
 
+def test_pickle_hook():
+    def forbidden_unknown_type():
+        raise RuntimeError("forbidden serial/deserial untrusted module")
+
+    def func_to_pk():
+        print("hello, world")
+        return 1234
+
+    with PickleHookOptions(serial_hook=forbidden_unknown_type):
+        with pytest.raises(RuntimeError):
+            serialize(func_to_pk)
+
+    content_bytes = serialize(func_to_pk)
+    with PickleHookOptions(deserial_hook=forbidden_unknown_type):
+        with pytest.raises(RuntimeError):
+            deserialize(*content_bytes)
+
+    deser_obj = deserialize(*content_bytes)
+    assert 1234 == deser_obj()
+
+
 def test_exceptions():
     try:
         raise ValueError("val")

maxframe/session.py

@@ -29,12 +29,11 @@ from weakref import ref
 
 from odps import ODPS
 
-from maxframe.core import TileableType
-from maxframe.lib.aio import Isolation, get_isolation, new_isolation, stop_isolation
-from maxframe.typing_ import ClientType
-from maxframe.utils import classproperty, implements, relay_future
-
 from .config import options
+from .core import TileableType
+from .lib.aio import Isolation, get_isolation, new_isolation, stop_isolation
+from .typing_ import ClientType
+from .utils import classproperty, implements, relay_future
 
 logger = logging.getLogger(__name__)
 
@@ -315,55 +314,6 @@ class AbstractAsyncSession(AbstractSession, metaclass=ABCMeta):
             web endpoint
         """
 
-    @abstractmethod
-    async def create_remote_object(
-        self, session_id: str, name: str, object_cls, *args, **kwargs
-    ):
-        """
-        Create remote object
-
-        Parameters
-        ----------
-        session_id : str
-            Session ID.
-        name : str
-        object_cls
-        args
-        kwargs
-
-        Returns
-        -------
-        actor_ref
-        """
-
-    @abstractmethod
-    async def get_remote_object(self, session_id: str, name: str):
-        """
-        Get remote object.
-
-        Parameters
-        ----------
-        session_id : str
-            Session ID.
-        name : str
-
-        Returns
-        -------
-        actor_ref
-        """
-
-    @abstractmethod
-    async def destroy_remote_object(self, session_id: str, name: str):
-        """
-        Destroy remote object.
-
-        Parameters
-        ----------
-        session_id : str
-            Session ID.
-        name : str
-        """
-
     async def stop_server(self):
         """
         Stop server.
@@ -722,23 +672,6 @@ class AsyncSession(AbstractAsyncSession):
     async def get_cluster_versions(self) -> List[str]:
         pass  # pragma: no cover
 
-    @implements(AbstractAsyncSession.create_remote_object)
-    @_delegate_to_isolated_session
-    async def create_remote_object(
-        self, session_id: str, name: str, object_cls, *args, **kwargs
-    ):
-        pass  # pragma: no cover
-
-    @implements(AbstractAsyncSession.get_remote_object)
-    @_delegate_to_isolated_session
-    async def get_remote_object(self, session_id: str, name: str):
-        pass  # pragma: no cover
-
-    @implements(AbstractAsyncSession.destroy_remote_object)
-    @_delegate_to_isolated_session
-    async def destroy_remote_object(self, session_id: str, name: str):
-        pass  # pragma: no cover
-
     @implements(AbstractAsyncSession.get_web_endpoint)
     @_delegate_to_isolated_session
     async def get_web_endpoint(self) -> Optional[str]:

maxframe/sperunner.py

@@ -0,0 +1,165 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import time
+from typing import Any, Dict
+
+import pandas as pd
+from odps import ODPS
+
+from .codegen import CodeGenResult
+from .codegen.spe.core import SPECodeContext
+from .config import option_context
+from .core import TileableGraph
+from .lib.compat import patch_pandas
+from .protocol import ResultInfo
+from .typing_ import PandasObjectTypes, TileableType
+from .utils import build_temp_table_name
+
+logger = logging.getLogger(__name__)
+
+
+class DAGCancelledError(Exception):
+    pass
+
+
+class SPEDagRunner:
+    def __init__(
+        self,
+        session_id: str,
+        subdag_id: str,
+        subdag: TileableGraph,
+        generated: CodeGenResult,
+        settings: Dict[str, Any],
+    ):
+        self._session_id = session_id
+        self._subdag_id = subdag_id
+        self._subdag = subdag
+        self._key_to_tileable = {t.key: t for t in subdag}
+        self._settings = settings
+
+        with option_context(self._settings) as session_options:
+            self._sql_hints = session_options.sql.settings
+
+        self._code = generated.code
+        self._constants = generated.constants
+        self._out_key_to_vars = generated.output_key_to_variables
+        self._out_key_to_infos = generated.output_key_to_result_infos
+        self._input_key_to_vars = generated.input_key_to_variables
+        self._mark_cancelled = False
+        self._odps = ODPS.from_environments()
+
+    def _get_sql_hints(self) -> Dict[str, str]:
+        hints = self._sql_hints.copy()
+        hints["odps.sql.type.system.odps2"] = True
+        return hints
+
+    @staticmethod
+    def _pre_process_pandas_data(data: PandasObjectTypes) -> PandasObjectTypes:
+        """
+        Sort the pandas dataset first to make sure all the tensor can process the
+        records in the same order.
+        """
+        if isinstance(data, (pd.DataFrame, pd.Series)):
+            data.sort_index(
+                axis=0,
+                ascending=True,
+                kind="quicksort",
+                inplace=True,
+                na_position="last",
+            )
+        elif isinstance(data, pd.Index):
+            data.sort_values(return_indexer=False, ascending=True, na_position="last")
+        return data
+
+    def fetch_data_by_tileable(self, t: TileableType) -> Any:
+        raise NotImplementedError
+
+    def store_data(self, key: str, value: Any) -> ResultInfo:
+        raise NotImplementedError
+
+    def run(self) -> Dict[str, ResultInfo]:
+        # make forward compatibility of new pandas methods
+        patch_pandas()
+
+        local_vars = self._constants
+        local_vars[SPECodeContext.logger_var] = logger
+        start_timestamp = time.time()
+        try:
+            # Fetch data
+            start_time = time.time()
+            for key, var_name in self._input_key_to_vars.items():
+                local_vars[var_name] = self.fetch_data_by_tileable(
+                    self._key_to_tileable[key]
+                )
+            logger.info(
+                "[%s][%s] fetch data costs: %.3f",
+                self._session_id,
+                self._subdag_id,
+                time.time() - start_time,
+            )
+
+            # Execute Python codes
+            start_time = time.time()
+            logger.info("Generated codes:\n--------\n%s\n--------", self._code)
+            exec(self._code, globals(), local_vars)
+            logger.info(
+                "[%s][%s] execute costs: %.3f",
+                self._session_id,
+                self._subdag_id,
+                time.time() - start_time,
+            )
+
+            # Store data
+            start_time = time.time()
+            result_dict = dict()
+            for key, var_name in self._out_key_to_vars.items():
+                result_dict[key] = self.store_data(key, local_vars[var_name])
+            logger.info(
+                "[%s][%s] store data costs: %.3f",
+                self._session_id,
+                self._subdag_id,
+                time.time() - start_time,
+            )
+            return result_dict
+        except Exception as ex:
+            local_vars.clear()
+            if (
+                not isinstance(ex, RuntimeError)
+                or not ex.args
+                or ex.args[0] != "CANCELLED"
+            ):
+                raise
+
+            drop_statements = [
+                f"DROP TABLE IF EXISTS {build_temp_table_name(self._session_id, key)};"
+                for key in self._out_key_to_vars.keys()
+            ]
+            if drop_statements:  # pragma: no branch
+                self._odps.run_sql(
+                    "\n".join(drop_statements), hints=self._get_sql_hints()
+                )
+            raise DAGCancelledError from None
+        finally:
+            logger.info(
+                "[%s][%s] run costs: %.3f",
+                self._session_id,
+                self._subdag_id,
+                time.time() - start_timestamp,
+            )
+
+    def mark_cancel(self):
+        self._constants["running"] = False
+        self._mark_cancelled = True

maxframe/tensor/__init__.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from . import fetch, random
+from . import extensions, fetch, linalg, random, special, statistics
 from .arithmetic import absolute
 from .arithmetic import absolute as abs
 from .arithmetic import (
@@ -121,16 +121,26 @@ from .datasource import (
     asarray,
     ascontiguousarray,
     asfortranarray,
+    diag,
+    diagflat,
     empty,
     empty_like,
+    eye,
     from_dataframe,
     full,
     full_like,
+    identity,
+    indices,
+    linspace,
+    meshgrid,
     ones,
     ones_like,
     scalar,
     tensor,
+    tril,
+    triu,
     zeros,
+    zeros_like,
 )
 from .indexing import (
     choose,
@@ -142,13 +152,31 @@ from .indexing import (
     take,
     unravel_index,
 )
-from .merge import concatenate, stack, vstack
+from .linalg.dot import dot
+from .linalg.inner import inner, innerproduct
+from .linalg.matmul import matmul
+from .linalg.tensordot import tensordot
+from .linalg.vdot import vdot
+from .merge import append, column_stack, concatenate, dstack, hstack, stack, vstack
 from .misc import (
+    astype,
     atleast_1d,
     atleast_2d,
     atleast_3d,
     broadcast_to,
+    copy,
+    diff,
+    in1d,
+    isin,
+    ndim,
+    ravel,
+    repeat,
+    searchsorted,
+    setdiff1d,
+    squeeze,
+    swapaxes,
     transpose,
+    trapezoid,
     unique,
     where,
 )
@@ -186,9 +214,14 @@ from .reduction import prod
 from .reduction import prod as product
 from .reduction import std, sum, var
 from .reshape import reshape
+from .sort import argsort, sort
+from .statistics import average, bincount, percentile, quantile
 from .ufunc import ufunc
 
 # isort: off
+# noinspection PyUnresolvedReferences
+from .lib.index_tricks import mgrid, ogrid, ndindex, r_, c_
+
 # noinspection PyUnresolvedReferences
 from numpy import (
     e,

maxframe/tensor/arithmetic/__init__.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 1999-2025 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,7 +15,7 @@
 from ...core import is_build_mode
 from .abs import TensorAbs, abs
 from .absolute import TensorAbsolute, absolute
-from .add import TensorAdd, add
+from .add import TensorAdd, TensorTreeAdd, add, tree_add
 from .angle import TensorAngle, angle
 from .arccos import TensorArccos, arccos
 from .arccosh import TensorArccosh, arccosh
@@ -89,7 +87,7 @@ from .mod import TensorMod
 from .mod import mod
 from .mod import mod as remainder
 from .modf import TensorModf, modf
-from .multiply import TensorMultiply, multiply
+from .multiply import TensorMultiply, TensorTreeMultiply, multiply, tree_multiply
 from .nan_to_num import TensorNanToNum, nan_to_num
 from .negative import TensorNegative, negative
 from .nextafter import TensorNextafter, nextafter

maxframe/tensor/arithmetic/abs.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 1999-2025 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

maxframe/tensor/arithmetic/absolute.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 1999-2025 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

maxframe/tensor/arithmetic/add.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 1999-2025 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,12 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import functools
+
 import numpy as np
 
 from ... import opcodes
+from ...serialization.serializables import BoolField
+from ..datasource import scalar
 from ..utils import infer_dtype
-from .core import TensorBinOp
-from .utils import arithmetic_operator
+from .core import TensorBinOp, TensorMultiOp
+from .utils import TreeReductionBuilder, arithmetic_operator
 
 
 @arithmetic_operator(sparse_mode="binary_and")
@@ -27,6 +29,10 @@ class TensorAdd(TensorBinOp):
     _op_type_ = opcodes.ADD
     _func_name = "add"
 
+    @classmethod
+    def _is_sparse_with_scalar(cls, scalar_val, lhs):
+        return isinstance(scalar_val, (int, float)) and scalar_val == 0
+
 
 @infer_dtype(np.add)
 def add(x1, x2, out=None, where=None, **kwargs):
@@ -80,3 +86,27 @@ def add(x1, x2, out=None, where=None, **kwargs):
 def radd(x1, x2, **kwargs):
     op = TensorAdd(**kwargs)
     return op.rcall(x1, x2)
+
+
+class TensorTreeAdd(TensorMultiOp):
+    _op_type_ = opcodes.TREE_ADD
+    _func_name = "add"
+
+    ignore_empty_input = BoolField("ignore_empty_input", default=False)
+
+    @classmethod
+    def _is_sparse(cls, *args):
+        if args and all(hasattr(x, "issparse") and x.issparse() for x in args):
+            return True
+        return False
+
+
+@infer_dtype(lambda *args: functools.reduce(np.add, args))
+def tree_add(*args, combine_size=None, **kwargs):
+    class MultiplyBuilder(TreeReductionBuilder):
+        def _build_reduction(self, inputs, final=False):
+            op = TensorTreeAdd(args=inputs, **kwargs)
+            return op(*inputs)
+
+    args = [scalar(a) if np.isscalar(a) else a for a in args]
+    return MultiplyBuilder(combine_size).build(args)

maxframe/tensor/arithmetic/angle.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 1999-2025 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

maxframe/tensor/arithmetic/arccos.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 1999-2025 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -22,10 +20,9 @@ from .core import TensorUnaryOp
 from .utils import arithmetic_operator
 
 
-@arithmetic_operator(sparse_mode="unary")
+@arithmetic_operator(sparse_mode="always_false")
 class TensorArccos(TensorUnaryOp):
     _op_type_ = opcodes.ARCCOS
-
     _func_name = "arccos"
 
 

maxframe/tensor/arithmetic/arccosh.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 1999-2025 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -22,7 +20,7 @@ from .core import TensorUnaryOp
 from .utils import arithmetic_operator
 
 
-@arithmetic_operator(sparse_mode="unary")
+@arithmetic_operator(sparse_mode="always_false")
 class TensorArccosh(TensorUnaryOp):
     _op_type_ = opcodes.ARCCOSH
     _func_name = "arccosh"

maxframe/tensor/arithmetic/arcsin.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 1999-2025 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

maxframe/tensor/arithmetic/arcsinh.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 1999-2025 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

maxframe/tensor/arithmetic/arctan.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 1999-2025 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

maxframe/tensor/arithmetic/arctan2.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 1999-2025 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

maxframe/tensor/arithmetic/arctanh.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 1999-2025 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

maxframe/tensor/arithmetic/around.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 1999-2025 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

maxframe/tensor/arithmetic/bitand.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 1999-2025 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

maxframe/tensor/arithmetic/bitor.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 1999-2025 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -22,7 +20,7 @@ from .core import TensorBinOp
 from .utils import arithmetic_operator
 
 
-@arithmetic_operator(sparse_mode="binary_or")
+@arithmetic_operator(sparse_mode="binary_and")
 class TensorBitor(TensorBinOp):
     _op_type_ = opcodes.BITOR
     _func_name = "bitwise_or"

maxframe/tensor/arithmetic/bitxor.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 1999-2025 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -22,7 +20,7 @@ from .core import TensorBinOp
 from .utils import arithmetic_operator
 
 
-@arithmetic_operator(sparse_mode="binary_or")
+@arithmetic_operator(sparse_mode="binary_and")
 class TensorBitxor(TensorBinOp):
     _op_type_ = opcodes.BITXOR
     _func_name = "bitwise_xor"

maxframe/tensor/arithmetic/cbrt.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 1999-2025 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

maxframe/tensor/arithmetic/ceil.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 1999-2025 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

maxframe/tensor/arithmetic/clip.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 1999-2025 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,11 +13,12 @@
 # limitations under the License.
 
 from numbers import Number
+from typing import List
 
 import numpy as np
 
 from ... import opcodes
-from ...core import ENTITY_TYPE
+from ...core import ENTITY_TYPE, EntityData
 from ...serialization.serializables import AnyField, KeyField
 from ..core import Tensor
 from ..datasource import tensor as astensor
@@ -35,16 +34,17 @@ class TensorClip(TensorOperator, TensorElementWise):
     a_max = AnyField("a_max", default=None)
     out = KeyField("out", default=None)
 
-    def _set_inputs(self, inputs):
-        super()._set_inputs(inputs)
-        inputs_iter = iter(self._inputs)
-        self.a = next(inputs_iter)
-        if isinstance(self.a_min, ENTITY_TYPE):
-            self.a_min = next(inputs_iter)
-        if isinstance(self.a_max, ENTITY_TYPE):
-            self.a_max = next(inputs_iter)
-        if getattr(self, "_out", None) is not None:
-            self.out = next(inputs_iter)
+    @classmethod
+    def _set_inputs(cls, op: "TensorClip", inputs: List[EntityData]):
+        super()._set_inputs(op, inputs)
+        inputs_iter = iter(op._inputs)
+        op.a = next(inputs_iter)
+        if isinstance(op.a_min, ENTITY_TYPE):
+            op.a_min = next(inputs_iter)
+        if isinstance(op.a_max, ENTITY_TYPE):
+            op.a_max = next(inputs_iter)
+        if getattr(op, "_out", None) is not None:
+            op.out = next(inputs_iter)
 
     def __call__(self, a, a_min, a_max, out=None):
         a = astensor(a)

maxframe/tensor/arithmetic/conj.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 1999-2025 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");