maxframe 0.1.0b2__cp37-cp37m-macosx_10_9_x86_64.whl → 0.1.0b3__cp37-cp37m-macosx_10_9_x86_64.whl

maxframe/codegen.py

@@ -17,7 +17,7 @@ import base64
 import dataclasses
 import logging
 from enum import Enum
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union
 
 from odps.types import OdpsSchema
 from odps.utils import camel_to_underline
@@ -30,6 +30,7 @@ from .odpsio import build_dataframe_table_meta
 from .odpsio.schema import pandas_to_odps_schema
 from .protocol import DataFrameTableMeta, ResultInfo
 from .serialization import PickleContainer
+from .serialization.serializables import Serializable, StringField
 from .typing_ import PandasObjectTypes
 from .udf import MarkedFunction
 
@@ -48,8 +49,11 @@ class CodeGenResult:
     constants: Dict[str, Any]
 
 
-class AbstractUDF(abc.ABC):
-    _session_id: str
+class AbstractUDF(Serializable):
+    _session_id: str = StringField("session_id")
+
+    def __init__(self, session_id: Optional[str] = None, **kw):
+        super().__init__(_session_id=session_id, **kw)
 
     @property
     def name(self) -> str:
@@ -74,7 +78,66 @@ class AbstractUDF(abc.ABC):
 
 class UserCodeMixin:
     @classmethod
-    def generate_pickled_codes(cls, code_to_pickle: Any) -> List[str]:
+    def obj_to_python_expr(cls, obj: Any = None) -> str:
+        """
+        Parameters
+        ----------
+        obj
+            The object to convert to python expr.
+        Returns
+        -------
+        str :
+            The str type content equals to the object when use in the python code directly.
+        """
+        if obj is None:
+            return "None"
+
+        if isinstance(obj, (int, float)):
+            return repr(obj)
+
+        if isinstance(obj, bool):
+            return "True" if obj else "False"
+
+        if isinstance(obj, bytes):
+            base64_bytes = base64.b64encode(obj)
+            return f"base64.b64decode({base64_bytes})"
+
+        if isinstance(obj, str):
+            return repr(obj)
+
+        if isinstance(obj, list):
+            return (
+                f"[{', '.join([cls.obj_to_python_expr(element) for element in obj])}]"
+            )
+
+        if isinstance(obj, dict):
+            items = (
+                f"{repr(key)}: {cls.obj_to_python_expr(value)}"
+                for key, value in obj.items()
+            )
+            return f"{{{', '.join(items)}}}"
+
+        if isinstance(obj, tuple):
+            return f"({', '.join([cls.obj_to_python_expr(sub_obj) for sub_obj in obj])}{',' if len(obj) == 1 else ''})"
+
+        if isinstance(obj, set):
+            return (
+                f"{{{', '.join([cls.obj_to_python_expr(sub_obj) for sub_obj in obj])}}}"
+                if obj
+                else "set()"
+            )
+
+        if isinstance(obj, PickleContainer):
+            return UserCodeMixin.generate_pickled_codes(obj, None)
+
+        raise ValueError(f"not support arg type {type(obj)}")
+
+    @classmethod
+    def generate_pickled_codes(
+        cls,
+        code_to_pickle: Any,
+        unpicked_data_var_name: Union[str, None] = "pickled_data",
+    ) -> str:
         """
         Generate pickled codes. The final pickled variable is called 'pickled_data'.
 
@@ -82,20 +145,20 @@ class UserCodeMixin:
         ----------
         code_to_pickle: Any
             The code to be pickled.
+        unpicked_data_var_name: str
+            The variables in code used to hold the loads object from the cloudpickle
 
         Returns
         -------
-        List[str] :
-            The code snippets of pickling, the final variable is called 'pickled_data'.
+        str :
+            The code snippets of pickling, the final variable is called 'pickled_data' by default.
         """
         pickled, buffers = cls.dump_pickled_data(code_to_pickle)
-        pickled = base64.b64encode(pickled)
-        buffers = [base64.b64encode(b) for b in buffers]
-        buffers_str = ", ".join(f"base64.b64decode(b'{b.decode()}')" for b in buffers)
-        return [
-            f"base64_data = base64.b64decode(b'{pickled.decode()}')",
-            f"pickled_data = cloudpickle.loads(base64_data, buffers=[{buffers_str}])",
-        ]
+        pickle_loads_expr = f"cloudpickle.loads({cls.obj_to_python_expr(pickled)}, buffers={cls.obj_to_python_expr(buffers)})"
+        if unpicked_data_var_name:
+            return f"{unpicked_data_var_name} = {pickle_loads_expr}"
+
+        return pickle_loads_expr
 
     @staticmethod
     def dump_pickled_data(
@@ -114,8 +177,9 @@ class UserCodeMixin:
 
 
 class BigDagCodeContext(metaclass=abc.ABCMeta):
-    def __init__(self, session_id: str = None):
+    def __init__(self, session_id: str = None, subdag_id: str = None):
         self._session_id = session_id
+        self._subdag_id = subdag_id
         self._tileable_key_to_variables = dict()
         self.constants = dict()
         self._data_table_meta_cache = dict()
@@ -142,10 +206,14 @@ class BigDagCodeContext(metaclass=abc.ABCMeta):
         except KeyError:
             var_name = self._tileable_key_to_variables[
                 tileable.key
-            ] = f"var_{self._next_var_id}"
-            self._next_var_id += 1
+            ] = self.next_var_name()
             return var_name
 
+    def next_var_name(self) -> str:
+        var_name = f"var_{self._next_var_id}"
+        self._next_var_id += 1
+        return var_name
+
     def get_odps_schema(
         self, data: PandasObjectTypes, unknown_as_string: bool = False
     ) -> OdpsSchema:
@@ -275,9 +343,10 @@ class BigDagCodeGenerator(metaclass=abc.ABCMeta):
     engine_priority: int = 0
     _extension_loaded = False
 
-    def __init__(self, session_id: str):
+    def __init__(self, session_id: str, subdag_id: str = None):
         self._session_id = session_id
-        self._context = self._init_context(session_id)
+        self._subdag_id = subdag_id
+        self._context = self._init_context(session_id, subdag_id)
 
     @classmethod
     def _load_engine_extensions(cls):
@@ -307,7 +376,7 @@ class BigDagCodeGenerator(metaclass=abc.ABCMeta):
         raise NotImplementedError
 
     @abc.abstractmethod
-    def _init_context(self, session_id: str) -> BigDagCodeContext:
+    def _init_context(self, session_id: str, subdag_id: str) -> BigDagCodeContext:
         raise NotImplementedError
 
     def _generate_comments(

maxframe/config/config.py

@@ -340,6 +340,12 @@ default_options.register_option(
     validator=is_integer,
     remote=True,
 )
+default_options.register_option(
+    "session.subinstance_priority",
+    None,
+    validator=any_validator(is_null, is_integer),
+    remote=True,
+)
 
 default_options.register_option("warn_duplicated_execution", False, validator=is_bool)
 default_options.register_option("dataframe.use_arrow_dtype", True, validator=is_bool)

maxframe/core/entity/executable.py

@@ -66,6 +66,7 @@ class DecrefRunner:
         if self._decref_thread:  # pragma: no branch
             self._queue.put_nowait((None, None, None))
             self._decref_thread.join(1)
+        self._decref_thread = None
 
     def put(self, key: str, session_ref: ref):
         if self._decref_thread is None:

maxframe/core/entity/objects.py

@@ -15,6 +15,7 @@
 from typing import Any, Dict
 
 from ...serialization.serializables import FieldTypes, ListField
+from ...utils import skip_na_call
 from .chunks import Chunk, ChunkData
 from .core import Entity
 from .executable import _ToObjectMixin
@@ -62,8 +63,8 @@ class ObjectData(TileableData, _ToObjectMixin):
     _chunks = ListField(
         "chunks",
         FieldTypes.reference(ObjectChunkData),
-        on_serialize=lambda x: [it.data for it in x] if x is not None else x,
-        on_deserialize=lambda x: [ObjectChunk(it) for it in x] if x is not None else x,
+        on_serialize=skip_na_call(lambda x: [it.data for it in x]),
+        on_deserialize=skip_na_call(lambda x: [ObjectChunk(it) for it in x]),
     )
 
     def __init__(self, op=None, nsplits=None, **kw):

maxframe/dataframe/__init__.py

@@ -39,6 +39,7 @@ from .datasource.read_odps_query import read_odps_query
 from .datasource.read_odps_table import read_odps_table
 from .datasource.read_parquet import read_parquet
 from .datastore.to_odps import to_odps_table
+from .groupby import NamedAgg
 from .initializer import DataFrame, Index, Series, read_pandas
 from .merge import concat, merge
 from .misc.cut import cut
@@ -52,7 +53,7 @@ from .reduction import CustomReduction, unique
 from .tseries.to_datetime import to_datetime
 
 try:
-    from pandas import NA, NamedAgg, Timestamp
+    from pandas import NA, Timestamp
 except ImportError:  # pragma: no cover
     pass
 

maxframe/dataframe/datasource/read_odps_query.py

@@ -46,7 +46,7 @@ _EXPLAIN_TASK_SCHEMA_REGEX = re.compile(
     r"In Task ([^:]+)[\S\s]+FS: output: ([^\n #]+)[\s\S]+schema:\s+([\S\s]+)$",
     re.MULTILINE,
 )
-_EXPLAIN_COLUMN_REGEX = re.compile(r"([^ ]+) \(([^)]+)\)(?:| AS ([^ ]+))(?:\n|$)")
+_EXPLAIN_COLUMN_REGEX = re.compile(r"([^\(]+) \(([^)]+)\)(?:| AS ([^ ]+))(?:\n|$)")
 
 
 @dataclasses.dataclass

maxframe/dataframe/datasource/read_odps_table.py

@@ -69,7 +69,7 @@ class DataFrameReadODPSTable(
         return getattr(self, "partition_spec", None)
 
     def get_columns(self):
-        return self.columns
+        return self.columns or list(self.dtypes.index)
 
     def set_pruned_columns(self, columns, *, keep_order=None):  # pragma: no cover
         self.columns = columns

maxframe/dataframe/datasource/tests/test_datasource.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 from collections import OrderedDict
 
 import numpy as np
@@ -33,6 +34,7 @@ from ..from_tensor import (
 )
 from ..index import from_pandas as from_pandas_index
 from ..index import from_tileable
+from ..read_odps_query import ColumnSchema, _resolve_task_sector
 from ..series import from_pandas as from_pandas_series
 
 ray = lazy_import("ray")
@@ -228,6 +230,7 @@ def test_from_odps_table():
     assert df.op.table_name == test_table.full_table_name
     assert df.index_value.name is None
     assert isinstance(df.index_value.value, IndexValue.RangeIndex)
+    assert df.op.get_columns() == ["col1", "col2", "col3"]
     pd.testing.assert_series_equal(
         df.dtypes,
         pd.Series(
@@ -247,6 +250,7 @@ def test_from_odps_table():
     assert df.op.table_name == test_table.full_table_name
     assert df.index_value.name is None
     assert isinstance(df.index_value.value, IndexValue.RangeIndex)
+    assert df.op.get_columns() == ["col1", "col2"]
     pd.testing.assert_series_equal(
         df.dtypes,
         pd.Series([np.dtype("O"), np.dtype("int64")], index=["col1", "col2"]),
@@ -257,6 +261,7 @@ def test_from_odps_table():
     assert df.index_value.name == "col1"
     assert isinstance(df.index_value.value, IndexValue.Index)
     assert df.index.dtype == np.dtype("O")
+    assert df.op.get_columns() == ["col2", "col3"]
     pd.testing.assert_series_equal(
         df.dtypes,
         pd.Series([np.dtype("int64"), np.dtype("float64")], index=["col2", "col3"]),
@@ -267,6 +272,7 @@ def test_from_odps_table():
 
     df = read_odps_table(test_parted_table, append_partitions=True)
     assert df.op.append_partitions is True
+    assert df.op.get_columns() == ["col1", "col2", "col3", "pt"]
     pd.testing.assert_series_equal(
         df.dtypes,
         pd.Series(
@@ -280,6 +286,7 @@ def test_from_odps_table():
     )
     assert df.op.append_partitions is True
     assert df.op.partitions == ["pt=20240103"]
+    assert df.op.get_columns() == ["col1", "col2", "pt"]
     pd.testing.assert_series_equal(
         df.dtypes,
         pd.Series(
@@ -377,3 +384,18 @@ def test_date_range():
     assert dr.index_value.is_unique == expected.is_unique
     assert dr.index_value.is_monotonic_increasing == expected.is_monotonic_increasing
     assert dr.name == expected.name
+
+
+def test_resolve_task_sector():
+    input_path = os.path.join(os.path.dirname(__file__), "test-data", "task-input.txt")
+    with open(input_path, "r") as f:
+        sector = f.read()
+    actual_sector = _resolve_task_sector("job0", sector)
+
+    assert actual_sector.job_name == "job0"
+    assert actual_sector.task_name == "M1"
+    assert actual_sector.output_target == "Screen"
+    assert len(actual_sector.schema) == 78
+    assert actual_sector.schema[0] == ColumnSchema("unnamed: 0", "bigint", "")
+    assert actual_sector.schema[1] == ColumnSchema("id", "bigint", "id_alias")
+    assert actual_sector.schema[2] == ColumnSchema("listing_url", "string", "")

maxframe/dataframe/groupby/__init__.py

@@ -14,6 +14,7 @@
 
 # noinspection PyUnresolvedReferences
 from ..core import DataFrameGroupBy, GroupBy, SeriesGroupBy
+from .core import NamedAgg
 
 
 def _install():

maxframe/dataframe/groupby/core.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from collections import namedtuple
+
 import pandas as pd
 
 from ... import opcodes
@@ -30,6 +32,9 @@ _GROUP_KEYS_NO_DEFAULT = pd_release_version >= (1, 5, 0)
 _default_group_keys = no_default if _GROUP_KEYS_NO_DEFAULT else True
 
 
+NamedAgg = namedtuple("NamedAgg", ["column", "aggfunc"])
+
+
 class DataFrameGroupByOperator(MapReduceOperator, DataFrameOperatorMixin):
     _op_type_ = opcodes.GROUPBY
 

maxframe/lib/aio/isolation.py

@@ -14,11 +14,14 @@
 
 import asyncio
 import atexit
+import itertools
 import threading
 from typing import Dict, Optional
 
 
 class Isolation:
+    _counter = itertools.count().__next__
+
     loop: asyncio.AbstractEventLoop
     _stopped: Optional[asyncio.Event]
     _thread: Optional[threading.Thread]
@@ -38,7 +41,9 @@ class Isolation:
 
     def start(self):
         if self._threaded:
-            self._thread = thread = threading.Thread(target=self._run)
+            self._thread = thread = threading.Thread(
+                name=f"IsolationThread-{self._counter()}", target=self._run
+            )
             thread.daemon = True
             thread.start()
             self._thread_ident = thread.ident

maxframe/protocol.py

@@ -46,6 +46,8 @@ BodyType = TypeVar("BodyType", bound="Serializable")
 
 
 class JsonSerializable(Serializable):
+    _ignore_non_existing_keys = True
+
     @classmethod
     def from_json(cls, serialized: dict) -> "JsonSerializable":
         raise NotImplementedError
@@ -245,6 +247,8 @@ class DagInfo(JsonSerializable):
         default_factory=dict,
     )
     error_info: Optional[ErrorInfo] = ReferenceField("error_info", default=None)
+    start_timestamp: Optional[float] = Float64Field("start_timestamp", default=None)
+    end_timestamp: Optional[float] = Float64Field("end_timestamp", default=None)
 
     @classmethod
     def from_json(cls, serialized: dict) -> "DagInfo":
@@ -265,7 +269,10 @@ class DagInfo(JsonSerializable):
             "dag_id": self.dag_id,
             "status": self.status.value,
             "progress": self.progress,
+            "start_timestamp": self.start_timestamp,
+            "end_timestamp": self.end_timestamp,
         }
+        ret = {k: v for k, v in ret.items() if v is not None}
         if self.tileable_to_result_infos:
             ret["tileable_to_result_infos"] = {
                 k: v.to_json() for k, v in self.tileable_to_result_infos.items()

maxframe/serialization/serializables/core.py

@@ -112,6 +112,7 @@ class Serializable(metaclass=SerializableMeta):
     __slots__ = ("__weakref__",)
 
     _cache_primitive_serial = False
+    _ignore_non_existing_keys = False
 
     _FIELDS: Dict[str, Field]
     _FIELD_ORDER: List[str]
@@ -128,7 +129,11 @@ class Serializable(metaclass=SerializableMeta):
         else:
             values = kwargs
         for k, v in values.items():
-            fields[k].set(self, v)
+            try:
+                fields[k].set(self, v)
+            except KeyError:
+                if not self._ignore_non_existing_keys:
+                    raise
 
     def __on_deserialize__(self):
         pass

maxframe/serialization/serializables/field.py

@@ -507,12 +507,14 @@ class ReferenceField(Field):
         tag: str,
         reference_type: Union[str, Type] = None,
         default: Any = no_default,
+        default_factory: Optional[Callable] = None,
         on_serialize: Callable[[Any], Any] = None,
         on_deserialize: Callable[[Any], Any] = None,
     ):
         super().__init__(
             tag,
             default=default,
+            default_factory=default_factory,
             on_serialize=on_serialize,
             on_deserialize=on_deserialize,
         )

maxframe/tensor/core.py

@@ -43,7 +43,7 @@ from ..serialization.serializables import (
     StringField,
     TupleField,
 )
-from ..utils import on_deserialize_shape, on_serialize_shape
+from ..utils import on_deserialize_shape, on_serialize_shape, skip_na_call
 from .utils import fetch_corner_data, get_chunk_slices
 
 logger = logging.getLogger(__name__)
@@ -181,8 +181,8 @@ class TensorData(HasShapeTileableData, _ExecuteAndFetchMixin):
     _chunks = ListField(
         "chunks",
         FieldTypes.reference(TensorChunkData),
-        on_serialize=lambda x: [it.data for it in x] if x is not None else x,
-        on_deserialize=lambda x: [TensorChunk(it) for it in x] if x is not None else x,
+        on_serialize=skip_na_call(lambda x: [it.data for it in x]),
+        on_deserialize=skip_na_call(lambda x: [TensorChunk(it) for it in x]),
     )
 
     def __init__(

maxframe/tests/test_codegen.py

@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+from typing import List, Tuple
+
+# 使用pytest生成单元测试
+import pytest
+
+from maxframe.codegen import UserCodeMixin
+from maxframe.lib import wrapped_pickle
+from maxframe.serialization.core import PickleContainer
+
+
+@pytest.mark.parametrize(
+    "input_obj, expected_output",
+    [
+        (None, "None"),
+        (10, "10"),
+        (3.14, "3.14"),
+        (True, "True"),
+        (False, "False"),
+        (b"hello", "base64.b64decode(b'aGVsbG8=')"),
+        ("hello", "'hello'"),
+        ([1, 2, 3], "[1, 2, 3]"),
+        ({"a": 1, "b": 2}, "{'a': 1, 'b': 2}"),
+        ((1, 2, 3), "(1, 2, 3)"),
+        ((1,), "(1,)"),
+        ((), "()"),
+        ({1, 2, 3}, "{1, 2, 3}"),
+        (set(), "set()"),
+    ],
+)
+def test_obj_to_python_expr(input_obj, expected_output):
+    assert UserCodeMixin.obj_to_python_expr(input_obj) == expected_output
+
+
+def test_obj_to_python_expr_custom_object():
+    class CustomClass:
+        def __init__(self, a: int, b: List[int], c: Tuple[int, int]):
+            self.a = a
+            self.b = b
+            self.c = c
+
+    custom_obj = CustomClass(1, [2, 3], (4, 5))
+    pickle_data = wrapped_pickle.dumps(custom_obj)
+    pickle_str = base64.b64encode(pickle_data)
+    custom_obj_pickle_container = PickleContainer([pickle_data])
+
+    # with class obj will not support currently
+    with pytest.raises(ValueError):
+        UserCodeMixin.obj_to_python_expr(custom_obj)
+
+    assert (
+        UserCodeMixin.obj_to_python_expr(custom_obj_pickle_container)
+        == f"cloudpickle.loads(base64.b64decode({pickle_str}), buffers=[])"
+    )

maxframe/tests/test_protocol.py

@@ -11,6 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+import json
 import time
 
 import pytest
@@ -29,28 +31,32 @@ from ..serialization import RemoteException
 from ..utils import deserialize_serializable, serialize_serializable
 
 
+def _json_round_trip(json_data: dict) -> dict:
+    return json.loads(json.dumps(json_data))
+
+
 def test_result_info_json_serialize():
-    ri = ResultInfo.from_json(ResultInfo().to_json())
+    ri = ResultInfo.from_json(_json_round_trip(ResultInfo().to_json()))
     assert type(ri) is ResultInfo
 
     ri = ODPSTableResultInfo(
         full_table_name="table_name", partition_specs=["pt=partition"]
     )
-    deserial_ri = ResultInfo.from_json(ri.to_json())
+    deserial_ri = ResultInfo.from_json(_json_round_trip(ri.to_json()))
     assert type(ri) is ODPSTableResultInfo
     assert ri.result_type == deserial_ri.result_type
     assert ri.full_table_name == deserial_ri.full_table_name
     assert ri.partition_specs == deserial_ri.partition_specs
 
     ri = ODPSTableResultInfo(full_table_name="table_name")
-    deserial_ri = ResultInfo.from_json(ri.to_json())
+    deserial_ri = ResultInfo.from_json(_json_round_trip(ri.to_json()))
     assert type(ri) is ODPSTableResultInfo
     assert ri.result_type == deserial_ri.result_type
     assert ri.full_table_name == deserial_ri.full_table_name
     assert ri.partition_specs == deserial_ri.partition_specs
 
     ri = ODPSVolumeResultInfo(volume_name="vol_name", volume_path="vol_path")
-    deserial_ri = ResultInfo.from_json(ri.to_json())
+    deserial_ri = ResultInfo.from_json(_json_round_trip(ri.to_json()))
     assert type(ri) is ODPSVolumeResultInfo
     assert ri.result_type == deserial_ri.result_type
     assert ri.volume_name == deserial_ri.volume_name
@@ -63,7 +69,7 @@ def test_error_info_json_serialize():
     except ValueError as ex:
         err_info = ErrorInfo.from_exception(ex)
 
-    deserial_err_info = ErrorInfo.from_json(err_info.to_json())
+    deserial_err_info = ErrorInfo.from_json(_json_round_trip(err_info.to_json()))
     assert deserial_err_info.error_messages == err_info.error_messages
     assert isinstance(deserial_err_info.raw_error_data, ValueError)
 
@@ -73,7 +79,7 @@ def test_error_info_json_serialize():
     with pytest.raises(RemoteException):
         mf_err_info.reraise()
 
-    deserial_err_info = ErrorInfo.from_json(mf_err_info.to_json())
+    deserial_err_info = ErrorInfo.from_json(_json_round_trip(mf_err_info.to_json()))
     assert isinstance(deserial_err_info.raw_error_data, ValueError)
     with pytest.raises(ValueError):
         deserial_err_info.reraise()
@@ -94,7 +100,9 @@ def test_dag_info_json_serialize():
         },
         error_info=err_info,
     )
-    deserial_info = DagInfo.from_json(info.to_json())
+    json_info = info.to_json()
+    json_info["non_existing_field"] = "non_existing"
+    deserial_info = DagInfo.from_json(_json_round_trip(json_info))
     assert deserial_info.session_id == info.session_id
     assert deserial_info.dag_id == info.dag_id
     assert deserial_info.status == info.status
@@ -121,7 +129,7 @@ def test_session_info_json_serialize():
         idle_timestamp=None,
         dag_infos={"test_dag_id": dag_info},
     )
-    deserial_info = SessionInfo.from_json(info.to_json())
+    deserial_info = SessionInfo.from_json(_json_round_trip(info.to_json()))
     assert deserial_info.session_id == info.session_id
     assert deserial_info.settings == info.settings
     assert deserial_info.start_timestamp == info.start_timestamp

maxframe/tests/utils.py

@@ -104,6 +104,7 @@ def run_app_in_thread(app_func):
         q = queue.Queue()
         exit_event = asyncio.Event(loop=app_loop)
         app_thread = Thread(
+            name="TestAppThread",
             target=app_thread_func,
             args=(app_loop, q, exit_event, args, kwargs),
         )

maxframe/utils.py

@@ -338,6 +338,14 @@ def deserialize_serializable(ser_serializable: bytes):
     return deserialize(header2, buffers2)
 
 
+def skip_na_call(func: Callable):
+    @functools.wraps(func)
+    def new_func(x):
+        return func(x) if x is not None else None
+
+    return new_func
+
+
 def url_path_join(*pieces):
     """Join components of url into a relative url
 
@@ -450,6 +458,9 @@ _ToThreadRetType = TypeVar("_ToThreadRetType")
 
 
 class ToThreadMixin:
+    _thread_pool_size = 1
+    _counter = itertools.count().__next__
+
     def __del__(self):
         if hasattr(self, "_pool"):
             kw = {"wait": False}
@@ -466,7 +477,10 @@ class ToThreadMixin:
         **kwargs,
     ) -> _ToThreadRetType:
         if not hasattr(self, "_pool"):
-            self._pool = concurrent.futures.ThreadPoolExecutor(1)
+            self._pool = concurrent.futures.ThreadPoolExecutor(
+                self._thread_pool_size,
+                thread_name_prefix=f"{type(self).__name__}Pool-{self._counter()}",
+            )
 
         task = asyncio.create_task(
             to_thread_pool(func, *args, **kwargs, pool=self._pool)

{maxframe-0.1.0b2.dist-info → maxframe-0.1.0b3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maxframe
-Version: 0.1.0b2
+Version: 0.1.0b3
 Summary: MaxFrame operator-based data analyze framework
 Requires-Dist: numpy >=1.19.0
 Requires-Dist: pandas >=1.0.0

{maxframe-0.1.0b2.dist-info → maxframe-0.1.0b3.dist-info}/RECORD

@@ -1,41 +1,41 @@
-maxframe-0.1.0b2.dist-info/RECORD,,
-maxframe-0.1.0b2.dist-info/WHEEL,sha256=HS_2vgrXyXO7N4Q-5VnuMAgG-fG_r8eemMgfU9Aw3Kk,110
-maxframe-0.1.0b2.dist-info/top_level.txt,sha256=O_LOO6KS5Y1ZKdmCjA9mzfg0-b1sB_P6Oy-hD8aSDfM,25
-maxframe-0.1.0b2.dist-info/METADATA,sha256=NeMSt0fZ-RjNm4v-FwVNTSA1Keglfyg5vWBBMTMCq_0,3043
+maxframe-0.1.0b3.dist-info/RECORD,,
+maxframe-0.1.0b3.dist-info/WHEEL,sha256=HS_2vgrXyXO7N4Q-5VnuMAgG-fG_r8eemMgfU9Aw3Kk,110
+maxframe-0.1.0b3.dist-info/top_level.txt,sha256=O_LOO6KS5Y1ZKdmCjA9mzfg0-b1sB_P6Oy-hD8aSDfM,25
+maxframe-0.1.0b3.dist-info/METADATA,sha256=056zhE5bxFHo_1CAOnpi1A2Ov1JI8hg435HsygF02lA,3043
 maxframe_client/conftest.py,sha256=7cwy2sFy5snEaxvtMvxfYFUnG6WtYC_9XxVrwJxOpcU,643
 maxframe_client/__init__.py,sha256=3b-z0oFVVwtIzVFBxOb9pw7gz4IhTSh4FiHtVgnxS4Q,724
 maxframe_client/fetcher.py,sha256=Ys_qu2qtniXuj9YSfeHvevdrAAEgm8k4YjyoZusdVmg,6813
 maxframe_client/clients/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
 maxframe_client/clients/spe.py,sha256=ArZMNQ7olicI4O1JO7CyRP7-hb60DF71ZKCTO0N39uE,3593
-maxframe_client/clients/framedriver.py,sha256=NtG3QjZpuMmxZl9IWJRAAzWLojaWn6k8vy0YRTq6hqU,4452
-maxframe_client/tests/test_session.py,sha256=Z86Zv2p1ECpiZDNK7YitRXoEIDmhNWbRJB0HSyX462Y,6238
+maxframe_client/clients/framedriver.py,sha256=Rn09529D2qBTgNGc0oCY0l7b3FgzT87TqS1nujGQaHw,4463
+maxframe_client/tests/test_session.py,sha256=75mxU5UTWjb1loQLo9BAkg7BzKuj4vYX49j4vP1-8fA,6307
 maxframe_client/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
 maxframe_client/tests/test_fetcher.py,sha256=q7kYCznM6WSxx9TCbHrxs7Zy1L2a5zu9D-Pi1XNgQzg,3516
-maxframe_client/session/task.py,sha256=lb_uNH-DKvL2VnGIJC449p5z3zR8eTyQ3B_xjH1MEOY,10842
+maxframe_client/session/task.py,sha256=z5j8qtBM6cs_lZrvfy4Ji3F3sVOhPOCr5r1RsNe7rN4,11102
 maxframe_client/session/graph.py,sha256=nwILNOIVaIf4E3xWffTAAlRsKRYU_zGW3oVO10du8Xw,4351
 maxframe_client/session/__init__.py,sha256=KPqhSlAJiuUz8TC-z5o7mHDVXzLSqWwrZ33zNni7piY,832
 maxframe_client/session/consts.py,sha256=R37BxDF3kgCy0qmDdwLaH5jB7mb7SzfYV6g9yHBKAwk,1344
 maxframe_client/session/odps.py,sha256=uuPk5rc1OpFk9PNFU1R6r4HQXOHAOU7ZVvbcdGk_N_s,16248
-maxframe_client/session/tests/test_task.py,sha256=MvizDb-WXbJn-cZ_e6rozO-VT95Expxu6ngyeD0TyZA,2617
+maxframe_client/session/tests/test_task.py,sha256=lDdw3gToaM3xSaRXEmHUoAo2h0id7t4v_VvpdKxQAao,3279
 maxframe_client/session/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
 maxframe/_utils.pyx,sha256=I4kmfhNr-xXK2ak22dr4Vwahzn-JmTaYctbL3y9_UBQ,17017
 maxframe/conftest.py,sha256=ZzwKhGp7LAVpzQYJkniwIUQqIegcaDQAhyzDyU2qld4,4264
 maxframe/opcodes.py,sha256=L1_GGenexNnD4dk1ueelgpAebHCC-cY7d--1QPhw8l4,9984
 maxframe/env.py,sha256=_K499f7giN7Iu9f39iI9p_naaEDoJ0rx8dInbzqFOVI,1402
 maxframe/mixin.py,sha256=HBAeWYGb7N6ZIgkA-YpkKiSY1GetcEVNTuMb0ieznBs,3524
-maxframe/protocol.py,sha256=8nt4L6C8CLiCU214gxKzwb4L_nxW-1vt368afZgQQgs,13822
+maxframe/protocol.py,sha256=LjjE6iw0ZVx82tBMbff4izkGuiJxRG0MTOaPYYpRL10,14190
 maxframe/session.py,sha256=TspAq0EUhi-7VQb1kAuQyjhpX8vMYqgdk4vzbGCkJcY,35344
 maxframe/__init__.py,sha256=YGnga-nYEppPoDZoZ5s64PZ0RYLaWtcYtmYSLTjKUBE,976
-maxframe/utils.py,sha256=U-MFtfKr0Np1PGsH9YhVomoKS-jzRrFGGSWkx9tYZGs,33555
+maxframe/utils.py,sha256=BrMJLO0-iPKABkNztXWpqiQWHoe-X75HGiCJr5aeles,33914
 maxframe/extension.py,sha256=4IzF9sPyaRoAzLud0iDLooOgcyq4QunXH0ki3q9Hn8I,2374
 maxframe/errors.py,sha256=nQZoLGdLJz-Uf9c2vUvQl08QMvRqtkBOhKfdPYRZA4o,690
-maxframe/_utils.cpython-37m-darwin.so,sha256=CQ-EQZ-mAN82XmssYNlXXqVCnLeWrXj9joPwcczP7po,488928
+maxframe/_utils.cpython-37m-darwin.so,sha256=OKe22C-cNEsoPdU-Hz69Z7Cqz3b-OWYvEFvTUa5O58g,488936
 maxframe/udf.py,sha256=tWOMTkNqGWw3ZpNC9wEU0GGNSBV8sV7E-Ye80DI28eg,2241
 maxframe/typing_.py,sha256=fzHETru3IOZAJwU9I7n_ib3wHuQRJ9XFVmAk7WpqkMo,1096
-maxframe/codegen.py,sha256=mqMIphJFN8r5jtiPrs51QuoBiZoOuYsHBcEeKnUBb-Y,13606
+maxframe/codegen.py,sha256=pSiGHoEo9YajFPHbFHvi7fGkiJmAQdBCe0mMXNOG6-U,15846
 maxframe/_utils.pxd,sha256=AhJ4vA_UqZqPshi5nvIZq1xgr80fhIVQ9dm5-UdkYJ8,1154
 maxframe/dataframe/arrays.py,sha256=RWzimUcrds5CsIlPausfJAkLUjcktBSSXwdXyUNKEtU,28752
-maxframe/dataframe/__init__.py,sha256=CttHAoUmkUi0o4qujJ29QwHdg2PKjcOGBT_D9eWkSIs,2092
+maxframe/dataframe/__init__.py,sha256=tpbt4OgW4hoU_1OsQu1WSxbpZD4YPXW_oT4xOxNPNqE,2112
 maxframe/dataframe/core.py,sha256=mJ5I_qBP8HDAzlwPnkj8RN4EyBHC8d5unbe2LC6HKXg,73641
 maxframe/dataframe/initializer.py,sha256=4BpZJB8bbyFnABUYWBrk_qzzrogEsWgFuU21Ma9IsjY,10264
 maxframe/dataframe/utils.py,sha256=qWRo51rcMTlo4mvZ8ZZq1zIF9CiAgU1qRtoCAaYrR34,44111
@@ -83,14 +83,14 @@ maxframe/dataframe/datasource/from_index.py,sha256=2061zsQn-BhyHTT0X9tE0JK8vLxQU
 maxframe/dataframe/datasource/dataframe.py,sha256=LxAKF4gBIHhnJQPuaAUdIEyMAq7HTfiEeNVls5n4I4A,2023
 maxframe/dataframe/datasource/series.py,sha256=QcYiBNcR8jjH6vdO6l6H9F46KHmlBqVCTI2tv9eyZ9w,1909
 maxframe/dataframe/datasource/__init__.py,sha256=C8EKsHTJi-1jvJUKIpZtMtsK-ZID3dtxL1voXnaltTs,640
-maxframe/dataframe/datasource/read_odps_query.py,sha256=t-kALzm2OXRHChCwRiTWPIp41o-q_v2xHNR9W6nm6Yk,9820
+maxframe/dataframe/datasource/read_odps_query.py,sha256=nj8l38S0iVAXgNXgzDFERO-HNp6lJ1GahImwDpEzXXw,9821
 maxframe/dataframe/datasource/core.py,sha256=ozFmDgw1og7nK9_jU-u3tLEq9pNbitN-8w8XWdbKkJ0,2687
 maxframe/dataframe/datasource/date_range.py,sha256=CDGpxDyjLwnb66j-MIiiTfXGXHGh5MLhEmj6x2riIlU,17244
-maxframe/dataframe/datasource/read_odps_table.py,sha256=2FIgD2N6IOFj9IbpwOCX_XYOmFpOugLY2Byl6-jtVsI,9018
+maxframe/dataframe/datasource/read_odps_table.py,sha256=VCqWxJswcjujVoUNXb2kbTkOZroMkFCg5n272Yn7ME4,9045
 maxframe/dataframe/datasource/read_parquet.py,sha256=9auOcy8snTxCOohgXZCUXfT_O39irdkBngZH5svgx0E,14531
 maxframe/dataframe/datasource/from_tensor.py,sha256=4viuN5SLLye7Xeb8kouOpm-osoQ2yEovWTDNPQuW8gE,14727
 maxframe/dataframe/datasource/from_records.py,sha256=WBYouYyg7m_8NJdN-yUWSfJlIpm6DVP3IMfLXZFugyI,3442
-maxframe/dataframe/datasource/tests/test_datasource.py,sha256=iSAg0W9lme45uBSOR77j0dR7dSevyrHpkM2VVONZGyU,13620
+maxframe/dataframe/datasource/tests/test_datasource.py,sha256=4O3N-XD-MpJxEQfILu4cS7gU82hqgS9g9gnDDEsw56k,14640
 maxframe/dataframe/datasource/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
 maxframe/dataframe/sort/sort_index.py,sha256=Hwbkm9x8kqGgXh4gMcAtYMDjYtt-S3CJXfYR9pN5Iqk,5412
 maxframe/dataframe/sort/__init__.py,sha256=Vt2Ynr7uAX51hLbQu93QeHiFH4D9_WJMO99KljpbO2U,1160
@@ -128,9 +128,9 @@ maxframe/dataframe/extensions/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0
 maxframe/dataframe/groupby/aggregation.py,sha256=BuhqZal6RLHkjvwJep86oT1a7rMqxxUAPxQ_dao6I6E,11953
 maxframe/dataframe/groupby/fill.py,sha256=JF3NxyXigZqg8ecKtLSndDmNMX8S6g_ChQR9JAK036E,4721
 maxframe/dataframe/groupby/cum.py,sha256=jdGQm7U-GosgHMfneHZC5Z2uraj6iBmSFOhqP3m18B0,3755
-maxframe/dataframe/groupby/__init__.py,sha256=TNb4WhAV0Q9d65bKMo_n3-bkPhAMj2YA6ZHalPUUeoU,3334
+maxframe/dataframe/groupby/__init__.py,sha256=nZkz1OAdYRj8qwQkUAZDax0pfCsUH_fprwuksS97vuc,3361
 maxframe/dataframe/groupby/getitem.py,sha256=kUcI9oIrjOcAHnho96Le9yEJxFydALsWbGpZfTtF8gY,3252
-maxframe/dataframe/groupby/core.py,sha256=AHO_6tX0fL1YkXLXUm2FjhPvxnmfmj-lkx711Uk5TPM,5980
+maxframe/dataframe/groupby/core.py,sha256=K1hg9jod6z3C65SYoidmEAd_k0Mear4l5IQuwNMjpxQ,6075
 maxframe/dataframe/groupby/transform.py,sha256=pY3WPA4gN8piYSTInncjnRdh8mi9FDQa00A-Pyaoil4,8586
 maxframe/dataframe/groupby/head.py,sha256=ZDkbSn3HuUR4GGkZJqo_fL-6KFJfs55aKXQkAh_0wvA,3266
 maxframe/dataframe/groupby/sample.py,sha256=IdoyzT-V5309txYvM_iaYKupsULfozMGwm1K3oihTf4,6935
@@ -267,13 +267,13 @@ maxframe/core/entity/tileables.py,sha256=b9jn_OQ-FQkbw7E7jMLjoJ4-VR7tBS8Mbx_j4iZ
 maxframe/core/entity/__init__.py,sha256=Hz_p6eTkrSdkT7YCo5aeGZ33tms5wwifMp4TeYsAVlw,1292
 maxframe/core/entity/core.py,sha256=t7Ex9Yb7A1h_XwyRG88Fx4ZOai-NQKi2luRVS_jFPEo,4018
 maxframe/core/entity/utils.py,sha256=IuNgFmBQFRioAA1hgZe6nTEggOmDY-iooZqncQQrV28,942
-maxframe/core/entity/executable.py,sha256=mkG6Ao6f4jAPhgGlPg4KOCwXIhdEZ5I69AEf3VF2iGQ,10903
+maxframe/core/entity/executable.py,sha256=HKXHXdPIyxg9i-OWmJxIY3KfXwX0x3xN9QcR5Xhc7dQ,10938
 maxframe/core/entity/output_types.py,sha256=uqApvFK8w6_aMxRets69dTwD1ndBDgVgqDCflyt9ubg,2645
-maxframe/core/entity/objects.py,sha256=3bhwSSzVD0o_UEzsXie-SuCie9y_mma9VDblpy3XmG8,3020
+maxframe/core/entity/objects.py,sha256=RMHLTGbIHZNxxX59lAuQydAKcR32qKleIYUqdElGS4E,3034
 maxframe/core/entity/fuse.py,sha256=47U6MHRzA2ZvUi-kJb7b3mC_gN07x3yebBgX2Jj7VZo,2277
 maxframe/core/entity/chunks.py,sha256=yNSLCWOpA_Z6aGr6ZI32dIJf3xPdRBWbvdsl8sTM3BE,2134
 maxframe/core/graph/__init__.py,sha256=rnsXwW0ouh1f7SVtq73-PzLE-MBM6Op_0l6J7b7wGRE,821
-maxframe/core/graph/core.cpython-37m-darwin.so,sha256=rPzFfp9ThfdOtXdwmrMtxbGYf4mITQ7zLuRGh5hLt-I,361144
+maxframe/core/graph/core.cpython-37m-darwin.so,sha256=BYD3I1sY48Dxnav_7JJF952cocb1923gQftQqN4t3-c,361120
 maxframe/core/graph/entity.py,sha256=56gjXyDXN-TTPm3AQOxuRVQbb_fguKFDL_Xm7i95XEk,5559
 maxframe/core/graph/core.pyx,sha256=ZJPx_MTOBMaX-6mns6tAiu-wrIBvRAKN44YAGTypJ1Y,15887
 maxframe/core/graph/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
@@ -294,7 +294,7 @@ maxframe/core/operator/fuse.py,sha256=0RGemF99gQCwV4aEk-K6T5KAGToO-487dFk8LyYDIZ
 maxframe/core/operator/base.py,sha256=nxuSKjbBzDrItM9PGmFo8RLwParazu525jMLWj0kXkM,15251
 maxframe/core/operator/tests/test_core.py,sha256=57aICnc5VLqdVK7icAORTWC81bSjBxeeVWIJcha9J_0,1691
 maxframe/core/operator/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
-maxframe/config/config.py,sha256=IqySITkIuIs0Gb6VME55Ox0WXOOT5yFwacC1gWLS83I,12897
+maxframe/config/config.py,sha256=3lQj99eMGg9MBW1gMJAdGFD88UEcdZf71sHgAHXASAk,13045
 maxframe/config/validators.py,sha256=2m9MrkjDUFiU4PPaWIw8tjwMaOy8AYmuJFqVnnY8IMY,1615
 maxframe/config/__init__.py,sha256=g5lN3nP2HTAXa6ExGxU1NwU1M9ulYPmAcsV-gU7nIW8,656
 maxframe/config/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
@@ -307,14 +307,14 @@ maxframe/serialization/arrow.py,sha256=VnGxNLU9UV_cUPTze43bEFCIbYLAOZnp2pAwVJbAI
 maxframe/serialization/__init__.py,sha256=9eSnoDww1uw2DAXEBBTB2atJQHzd-38XVxrCkoaypxA,921
 maxframe/serialization/maxframe_objects.py,sha256=R9WEjbHL0Kr56OGkYDU9fcGi7gII6fGlXhi6IyihTsM,1365
 maxframe/serialization/numpy.py,sha256=8_GSo45l_eNoMn4NAGEb9NLXY_9i4tf9KK4EzG0mKpA,3213
-maxframe/serialization/core.cpython-37m-darwin.so,sha256=0pF_PpwHPD_wx-TD5o4IOEwYPRymX3nvK-INeEwEXkI,695456
+maxframe/serialization/core.cpython-37m-darwin.so,sha256=XN9VpQtHGwy82OmscDwRd3LrTrmusQaQ4PBOohTCFM0,695464
 maxframe/serialization/scipy.py,sha256=hP0fAW0di9UgJrGtANB2S8hLDbFBtR8p5NDqAMt5rDI,2427
 maxframe/serialization/core.pyx,sha256=AATN47RdBTq2zg7--3xX2VHyAZSvoAuYRt7B7gEgKPE,33984
 maxframe/serialization/tests/test_serial.py,sha256=Wj_I6CBQMaOtE8WtqdUaBoU8FhBOihht6SfeHOJV-zU,12511
 maxframe/serialization/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
-maxframe/serialization/serializables/field.py,sha256=OjQyF667xANnNTVpQ34etd2LTNZrNUW_1K5lYuVWI1U,15918
+maxframe/serialization/serializables/field.py,sha256=atVgX-9rsVG1fTev7vjQArVwIEaCRjXoSEjpQ3mh6bA,16015
 maxframe/serialization/serializables/__init__.py,sha256=_wyFZF5QzSP32wSXlXHEPl98DN658I66WamP8XPJy0c,1351
-maxframe/serialization/serializables/core.py,sha256=a_6_UgomggS5ty5LXycnAphjFiqA0nEqgNUOQGN04pk,8796
+maxframe/serialization/serializables/core.py,sha256=xlqVUlBK3aLTavHLWHg4JXUTaBGzSuM7t-XHahB8et4,8965
 maxframe/serialization/serializables/field_type.py,sha256=Feh09hu8XyaxS5MaJ4za_pcvqJVuMkOeGxwQ9OuJw6I,14865
 maxframe/serialization/serializables/tests/test_field_type.py,sha256=T3ebXbUkKveC9Pq1nIl85e4eYascFeJ52d0REHbz5jo,4381
 maxframe/serialization/serializables/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
@@ -330,14 +330,15 @@ maxframe/odpsio/tests/test_schema.py,sha256=yss1ly55ErYse95XMFq2s_GWL8UnwZga5RyN
 maxframe/odpsio/tests/test_arrow.py,sha256=SQ9EmI9_VOOC8u6Rg6nh3IPC2fPbLvJ9HwtpMNDRhL8,3106
 maxframe/odpsio/tests/test_volumeio.py,sha256=UEqFANuPKyFtlIh2JNi-LoixH52bxsgHdxu3himnEvs,3022
 maxframe/tests/test_utils.py,sha256=xaAoURr5NOJUTY0XVa2H8qOStcEH5UQSXItkatHFxFE,11977
-maxframe/tests/test_protocol.py,sha256=GnA-czIKHvNqZCBbgnWQZ1CP06_SorTEvGZ9VHUGvK4,4857
+maxframe/tests/test_protocol.py,sha256=t11yxh4_gWxxCuk09zo3pn9Nn96DBBQTBt12ewKDwLQ,5187
 maxframe/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
-maxframe/tests/utils.py,sha256=DHpzJLs-U6xiv5jSHe0m1D2b5bk7WLA7fa2-PY4HpOg,4534
+maxframe/tests/utils.py,sha256=wJtSFXt3BD4i5zdO4JBQk_kNAxrtyGLro0jodCA4xuY,4568
+maxframe/tests/test_codegen.py,sha256=GMrnpSb2eyB_nmuv8-_p47Kw877ElKS3BP52SpqZNIQ,2208
 maxframe/lib/wrapped_pickle.py,sha256=xJa0wI-GsBZFKQpVnlh_hZBlQ2u1D8VO2aBIW7VOdP4,3810
 maxframe/lib/version.py,sha256=yQ6HkDOvU9X1rpI49auh-qku2g7gIiztgEH6v1urOrk,18321
 maxframe/lib/compression.py,sha256=k9DSrl_dNBsn5azLjBdL5B4WZ6eNvmCrdMbcF1G7JSc,1442
 maxframe/lib/__init__.py,sha256=CzfbLNqqm1yR1i6fDwCd4h1ptuKVDbURFVCb0ra7QNc,642
-maxframe/lib/mmh3.cpython-37m-darwin.so,sha256=x5pjOTvSfsLeduNnNaJNhZ2a6LkNiU51loSeFgp7nt8,37904
+maxframe/lib/mmh3.cpython-37m-darwin.so,sha256=WAmuMKlVy42_JaGyRDqYEHrWftb3Uk48e2uaDTK8ugE,37904
 maxframe/lib/functools_compat.py,sha256=PMSkct9GIbzq-aBwTnggrOLNfLh4xQnYTIFMPblzCUA,2616
 maxframe/lib/mmh3_src/mmh3module.cpp,sha256=9J9eA42eKWTl546fvfQPNuIM3B2jpWSADpgIw3tr2jg,11604
 maxframe/lib/mmh3_src/MurmurHash3.h,sha256=lg5uXUFyMBge2BWRn0FgrqaCFCMfDWoTXD4PQtjHrMA,1263
@@ -346,7 +347,7 @@ maxframe/lib/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZH
 maxframe/lib/tests/test_wrapped_pickle.py,sha256=oz1RLwHSZstXgw4caNeaD0ZgQZvkzDLsx7hFN-NvP7U,1524
 maxframe/lib/cython/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
 maxframe/lib/cython/libcpp.pxd,sha256=2o9HWtyCMtN9xk7WH_mq1i89IbMPwfZA8yHETjRALXs,1100
-maxframe/lib/aio/isolation.py,sha256=p2QuFBVe1ktX8uVFbu6_U1yNzbhBNpFZ9f4A6-Y6XHo,2628
+maxframe/lib/aio/isolation.py,sha256=2nA16GdOXEUNVVdxQXbmU4YvY6wPG2oSV4z1x9uW6Gw,2761
 maxframe/lib/aio/__init__.py,sha256=1_nx7d5AqXRwa7ODzVfL8gH-tsDAH4YyY-JFPC8TA6w,936
 maxframe/lib/aio/_threads.py,sha256=tr7FYViGT7nyR7HRw3vE3W-9r3-dZ1IP_Kbhe9sgqpw,1328
 maxframe/lib/aio/file.py,sha256=aZF8NkkccsVsOniWMBiPqPSk48bb7zGRPB2BegWRaqM,2012
@@ -386,7 +387,7 @@ maxframe/lib/tblib/cpython.py,sha256=FQ0f6WTQyQHoMRhgPqrA0y0Ygxlbj5IC53guxA4h9Cw
 maxframe/lib/tblib/decorators.py,sha256=bcllK3kVuPnj6SNZGmlJGxTK0ovdt7TJDXrhA4UE5sQ,1063
 maxframe/tensor/array_utils.py,sha256=259vG4SjyhiheARCZeEnfJdZjoojyrELn41oRcyAELs,4943
 maxframe/tensor/__init__.py,sha256=-kir8LUsXCDGcc7YdKqWgNEHSrgU_HE5uPam0jLLP6g,3511
-maxframe/tensor/core.py,sha256=x9HRQbMjhXJTtPQjKhkN42qZaPQIR7uvkMkOpqUspL8,21920
+maxframe/tensor/core.py,sha256=Ojxaf5b8sJ6ZZGezyFHQJ5XsSpUrBOnZgFeUQgpVJpI,21914
 maxframe/tensor/utils.py,sha256=bwVN0iuVic1tpFai6Hk-1tQLqckQ2IYS7yZKMTcOU1I,22914
 maxframe/tensor/operators.py,sha256=iGkDIRz152gXrPb5JbqOvXngpq3QaCg-aNO4gHZPLN0,3461
 maxframe/tensor/statistics/quantile.py,sha256=UFzTmBwgNL7k_QOJ84qPfycQrW8MyOa1gcp-uFsylIY,9484

maxframe_client/clients/framedriver.py

@@ -14,6 +14,7 @@
 
 from typing import Any, Dict, List
 
+import msgpack
 from tornado import httpclient
 
 from maxframe.core import TileableGraph
@@ -28,7 +29,6 @@ from maxframe.protocol import (
 )
 from maxframe.typing_ import TimeoutType
 from maxframe.utils import (
-    deserialize_serializable,
     format_timeout_params,
     serialize_serializable,
     wait_http_response,
@@ -47,12 +47,12 @@ class FrameDriverClient:
         resp = await httpclient.AsyncHTTPClient().fetch(
             req_url, method="POST", body=serialize_serializable(req_body)
         )
-        return deserialize_serializable(resp.body).body
+        return SessionInfo.from_json(msgpack.loads(resp.body))
 
     async def get_session(self, session_id: str) -> SessionInfo:
         req_url = f"{self._endpoint}/api/sessions/{session_id}"
         resp = await httpclient.AsyncHTTPClient().fetch(req_url, method="GET")
-        return deserialize_serializable(resp.body).body
+        return SessionInfo.from_json(msgpack.loads(resp.body))
 
     async def delete_session(self, session_id: str):
         req_url = f"{self._endpoint}/api/sessions/{session_id}"
@@ -71,12 +71,12 @@ class FrameDriverClient:
             method="POST",
             body=serialize_serializable(ProtocolBody(body=req_body)),
         )
-        return deserialize_serializable(resp.body).body
+        return DagInfo.from_json(msgpack.loads(resp.body))
 
     async def get_dag_info(self, session_id: str, dag_id: str) -> DagInfo:
         req_url = f"{self._endpoint}/api/sessions/{session_id}/dags/{dag_id}"
         resp = await httpclient.AsyncHTTPClient().fetch(req_url, method="GET")
-        return deserialize_serializable(resp.body).body
+        return DagInfo.from_json(msgpack.loads(resp.body))
 
     async def wait_dag(self, session_id: str, dag_id: str, timeout: TimeoutType = None):
         query_part = format_timeout_params(timeout)
@@ -87,7 +87,7 @@ class FrameDriverClient:
             resp = await wait_http_response(
                 req_url, method="GET", request_timeout=timeout
             )
-            info = deserialize_serializable(resp.body).body
+            info = DagInfo.from_json(msgpack.loads(resp.body))
         except TimeoutError:
             info = await self.get_dag_info(session_id, dag_id)
         return info
@@ -103,7 +103,7 @@ class FrameDriverClient:
             resp = await wait_http_response(
                 req_url, method="DELETE", request_timeout=timeout
             )
-            info = deserialize_serializable(resp.body).body
+            info = DagInfo.from_json(msgpack.loads(resp.body))
         except TimeoutError:
             info = await self.get_dag_info(session_id, dag_id)
         return info

maxframe_client/session/task.py

@@ -112,13 +112,19 @@ class MaxFrameInstanceCaller(MaxFrameServiceCaller):
         odps_entry: ODPS,
         task_name: Optional[str] = None,
         project: Optional[str] = None,
-        priority: Optional[str] = None,
+        priority: Optional[int] = None,
         running_cluster: Optional[str] = None,
         nested_instance_id: Optional[str] = None,
         major_version: Optional[str] = None,
         output_format: Optional[str] = None,
         **kwargs,
     ):
+        if callable(odps_options.get_priority):
+            default_priority = odps_options.get_priority(odps_entry)
+        else:
+            default_priority = odps_options.priority
+        priority = priority if priority is not None else default_priority
+
         self._odps_entry = odps_entry
         self._task_name = task_name
         self._project = project
@@ -126,6 +132,7 @@ class MaxFrameInstanceCaller(MaxFrameServiceCaller):
         self._running_cluster = running_cluster
         self._major_version = major_version
         self._output_format = output_format or MAXFRAME_OUTPUT_MSGPACK_FORMAT
+
         if nested_instance_id is None:
             self._nested = False
             self._instance = None

maxframe_client/session/tests/test_task.py

@@ -18,6 +18,7 @@ import os
 import mock
 from defusedxml import ElementTree
 from odps import ODPS
+from odps import options as odps_options
 
 from ...session.consts import MAXFRAME_OUTPUT_JSON_FORMAT
 from ...session.task import MaxFrameInstanceCaller, MaxFrameTask
@@ -27,17 +28,20 @@ expected_file_dir = os.path.join(os.path.dirname(__file__), "expected-data")
 
 def test_maxframe_instance_caller_creating_session():
     o = ODPS.from_environments()
-    task_caller = MaxFrameInstanceCaller(
-        odps_entry=o,
-        task_name="task_test",
-        major_version="test_version",
-        output_format=MAXFRAME_OUTPUT_JSON_FORMAT,
-        priority="100",
-        running_cluster="test_cluster",
-    )
+
+    def create_caller(**kwargs):
+        kw = dict(
+            odps_entry=o,
+            task_name="task_test",
+            major_version="test_version",
+            output_format=MAXFRAME_OUTPUT_JSON_FORMAT,
+            running_cluster="test_cluster",
+        )
+        kw.update(**kwargs)
+        return MaxFrameInstanceCaller(**kw)
 
     def mock_create(self, task: MaxFrameTask, priority=None, running_cluster=None):
-        assert priority == "100"
+        assert priority == 100
         assert running_cluster == "test_cluster"
         root = ElementTree.parse(
             os.path.join(expected_file_dir, "create_session.xml")
@@ -62,6 +66,20 @@ def test_maxframe_instance_caller_creating_session():
         target="maxframe_client.session.task.MaxFrameInstanceCaller",
         _wait_instance_task_ready=mock.DEFAULT,
         get_session=mock.DEFAULT,
-    ):
-        with mock.patch("odps.models.instances.BaseInstances.create", mock_create):
+    ), mock.patch("odps.models.instances.BaseInstances.create", mock_create):
+        task_caller = create_caller(priority=100)
+        task_caller.create_session()
+
+        old_priority = odps_options.priority
+        old_get_priority = odps_options.get_priority
+        try:
+            task_caller = create_caller(priority=100)
+            odps_options.priority = 100
+            task_caller.create_session()
+
+            odps_options.priority = None
+            odps_options.get_priority = lambda _: 100
             task_caller.create_session()
+        finally:
+            odps_options.priority = old_priority
+            odps_options.get_priority = old_get_priority

maxframe_client/tests/test_session.py

@@ -24,6 +24,7 @@ from odps import ODPS
 import maxframe.dataframe as md
 import maxframe.remote as mr
 from maxframe.core import ExecutableTuple, TileableGraph
+from maxframe.lib.aio import stop_isolation
 from maxframe.protocol import ResultInfo
 from maxframe.serialization import RemoteException
 from maxframe.session import new_session
@@ -52,6 +53,7 @@ def start_mock_session(framedriver_app):  # noqa: F811
             time.sleep(5)  # Wait for temp table deleted
         else:
             session.reset_default()
+        stop_isolation()
 
 
 def test_simple_run_dataframe(start_mock_session):