maxframe 0.1.0b2__cp39-cp39-win_amd64.whl → 0.1.0b3__cp39-cp39-win_amd64.whl

maxframe/codegen.py

@@ -17,7 +17,7 @@ import base64
 import dataclasses
 import logging
 from enum import Enum
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union
 
 from odps.types import OdpsSchema
 from odps.utils import camel_to_underline
@@ -30,6 +30,7 @@ from .odpsio import build_dataframe_table_meta
 from .odpsio.schema import pandas_to_odps_schema
 from .protocol import DataFrameTableMeta, ResultInfo
 from .serialization import PickleContainer
+from .serialization.serializables import Serializable, StringField
 from .typing_ import PandasObjectTypes
 from .udf import MarkedFunction
 
@@ -48,8 +49,11 @@ class CodeGenResult:
     constants: Dict[str, Any]
 
 
-class AbstractUDF(abc.ABC):
-    _session_id: str
+class AbstractUDF(Serializable):
+    _session_id: str = StringField("session_id")
+
+    def __init__(self, session_id: Optional[str] = None, **kw):
+        super().__init__(_session_id=session_id, **kw)
 
     @property
     def name(self) -> str:
@@ -74,7 +78,66 @@ class AbstractUDF(abc.ABC):
 
 class UserCodeMixin:
     @classmethod
-    def generate_pickled_codes(cls, code_to_pickle: Any) -> List[str]:
+    def obj_to_python_expr(cls, obj: Any = None) -> str:
+        """
+        Parameters
+        ----------
+        obj
+            The object to convert to python expr.
+        Returns
+        -------
+        str :
+            The str type content equals to the object when use in the python code directly.
+        """
+        if obj is None:
+            return "None"
+
+        if isinstance(obj, (int, float)):
+            return repr(obj)
+
+        if isinstance(obj, bool):
+            return "True" if obj else "False"
+
+        if isinstance(obj, bytes):
+            base64_bytes = base64.b64encode(obj)
+            return f"base64.b64decode({base64_bytes})"
+
+        if isinstance(obj, str):
+            return repr(obj)
+
+        if isinstance(obj, list):
+            return (
+                f"[{', '.join([cls.obj_to_python_expr(element) for element in obj])}]"
+            )
+
+        if isinstance(obj, dict):
+            items = (
+                f"{repr(key)}: {cls.obj_to_python_expr(value)}"
+                for key, value in obj.items()
+            )
+            return f"{{{', '.join(items)}}}"
+
+        if isinstance(obj, tuple):
+            return f"({', '.join([cls.obj_to_python_expr(sub_obj) for sub_obj in obj])}{',' if len(obj) == 1 else ''})"
+
+        if isinstance(obj, set):
+            return (
+                f"{{{', '.join([cls.obj_to_python_expr(sub_obj) for sub_obj in obj])}}}"
+                if obj
+                else "set()"
+            )
+
+        if isinstance(obj, PickleContainer):
+            return UserCodeMixin.generate_pickled_codes(obj, None)
+
+        raise ValueError(f"not support arg type {type(obj)}")
+
+    @classmethod
+    def generate_pickled_codes(
+        cls,
+        code_to_pickle: Any,
+        unpicked_data_var_name: Union[str, None] = "pickled_data",
+    ) -> str:
         """
         Generate pickled codes. The final pickled variable is called 'pickled_data'.
 
@@ -82,20 +145,20 @@ class UserCodeMixin:
         ----------
         code_to_pickle: Any
             The code to be pickled.
+        unpicked_data_var_name: str
+            The variables in code used to hold the loads object from the cloudpickle
 
         Returns
         -------
-        List[str] :
-            The code snippets of pickling, the final variable is called 'pickled_data'.
+        str :
+            The code snippets of pickling, the final variable is called 'pickled_data' by default.
         """
         pickled, buffers = cls.dump_pickled_data(code_to_pickle)
-        pickled = base64.b64encode(pickled)
-        buffers = [base64.b64encode(b) for b in buffers]
-        buffers_str = ", ".join(f"base64.b64decode(b'{b.decode()}')" for b in buffers)
-        return [
-            f"base64_data = base64.b64decode(b'{pickled.decode()}')",
-            f"pickled_data = cloudpickle.loads(base64_data, buffers=[{buffers_str}])",
-        ]
+        pickle_loads_expr = f"cloudpickle.loads({cls.obj_to_python_expr(pickled)}, buffers={cls.obj_to_python_expr(buffers)})"
+        if unpicked_data_var_name:
+            return f"{unpicked_data_var_name} = {pickle_loads_expr}"
+
+        return pickle_loads_expr
 
     @staticmethod
     def dump_pickled_data(
@@ -114,8 +177,9 @@ class UserCodeMixin:
 
 
 class BigDagCodeContext(metaclass=abc.ABCMeta):
-    def __init__(self, session_id: str = None):
+    def __init__(self, session_id: str = None, subdag_id: str = None):
         self._session_id = session_id
+        self._subdag_id = subdag_id
         self._tileable_key_to_variables = dict()
         self.constants = dict()
         self._data_table_meta_cache = dict()
@@ -142,10 +206,14 @@ class BigDagCodeContext(metaclass=abc.ABCMeta):
         except KeyError:
             var_name = self._tileable_key_to_variables[
                 tileable.key
-            ] = f"var_{self._next_var_id}"
-            self._next_var_id += 1
+            ] = self.next_var_name()
             return var_name
 
+    def next_var_name(self) -> str:
+        var_name = f"var_{self._next_var_id}"
+        self._next_var_id += 1
+        return var_name
+
     def get_odps_schema(
         self, data: PandasObjectTypes, unknown_as_string: bool = False
     ) -> OdpsSchema:
@@ -275,9 +343,10 @@ class BigDagCodeGenerator(metaclass=abc.ABCMeta):
     engine_priority: int = 0
     _extension_loaded = False
 
-    def __init__(self, session_id: str):
+    def __init__(self, session_id: str, subdag_id: str = None):
         self._session_id = session_id
-        self._context = self._init_context(session_id)
+        self._subdag_id = subdag_id
+        self._context = self._init_context(session_id, subdag_id)
 
     @classmethod
     def _load_engine_extensions(cls):
@@ -307,7 +376,7 @@ class BigDagCodeGenerator(metaclass=abc.ABCMeta):
         raise NotImplementedError
 
     @abc.abstractmethod
-    def _init_context(self, session_id: str) -> BigDagCodeContext:
+    def _init_context(self, session_id: str, subdag_id: str) -> BigDagCodeContext:
         raise NotImplementedError
 
     def _generate_comments(

maxframe/config/config.py

@@ -340,6 +340,12 @@ default_options.register_option(
     validator=is_integer,
     remote=True,
 )
+default_options.register_option(
+    "session.subinstance_priority",
+    None,
+    validator=any_validator(is_null, is_integer),
+    remote=True,
+)
 
 default_options.register_option("warn_duplicated_execution", False, validator=is_bool)
 default_options.register_option("dataframe.use_arrow_dtype", True, validator=is_bool)

maxframe/core/entity/executable.py

@@ -66,6 +66,7 @@ class DecrefRunner:
         if self._decref_thread:  # pragma: no branch
             self._queue.put_nowait((None, None, None))
             self._decref_thread.join(1)
+        self._decref_thread = None
 
     def put(self, key: str, session_ref: ref):
         if self._decref_thread is None:

maxframe/core/entity/objects.py

@@ -15,6 +15,7 @@
 from typing import Any, Dict
 
 from ...serialization.serializables import FieldTypes, ListField
+from ...utils import skip_na_call
 from .chunks import Chunk, ChunkData
 from .core import Entity
 from .executable import _ToObjectMixin
@@ -62,8 +63,8 @@ class ObjectData(TileableData, _ToObjectMixin):
     _chunks = ListField(
         "chunks",
         FieldTypes.reference(ObjectChunkData),
-        on_serialize=lambda x: [it.data for it in x] if x is not None else x,
-        on_deserialize=lambda x: [ObjectChunk(it) for it in x] if x is not None else x,
+        on_serialize=skip_na_call(lambda x: [it.data for it in x]),
+        on_deserialize=skip_na_call(lambda x: [ObjectChunk(it) for it in x]),
     )
 
     def __init__(self, op=None, nsplits=None, **kw):

maxframe/dataframe/__init__.py

@@ -39,6 +39,7 @@ from .datasource.read_odps_query import read_odps_query
 from .datasource.read_odps_table import read_odps_table
 from .datasource.read_parquet import read_parquet
 from .datastore.to_odps import to_odps_table
+from .groupby import NamedAgg
 from .initializer import DataFrame, Index, Series, read_pandas
 from .merge import concat, merge
 from .misc.cut import cut
@@ -52,7 +53,7 @@ from .reduction import CustomReduction, unique
 from .tseries.to_datetime import to_datetime
 
 try:
-    from pandas import NA, NamedAgg, Timestamp
+    from pandas import NA, Timestamp
 except ImportError:  # pragma: no cover
     pass
 

maxframe/dataframe/datasource/read_odps_query.py

@@ -46,7 +46,7 @@ _EXPLAIN_TASK_SCHEMA_REGEX = re.compile(
     r"In Task ([^:]+)[\S\s]+FS: output: ([^\n #]+)[\s\S]+schema:\s+([\S\s]+)$",
     re.MULTILINE,
 )
-_EXPLAIN_COLUMN_REGEX = re.compile(r"([^ ]+) \(([^)]+)\)(?:| AS ([^ ]+))(?:\n|$)")
+_EXPLAIN_COLUMN_REGEX = re.compile(r"([^\(]+) \(([^)]+)\)(?:| AS ([^ ]+))(?:\n|$)")
 
 
 @dataclasses.dataclass

maxframe/dataframe/datasource/read_odps_table.py

@@ -69,7 +69,7 @@ class DataFrameReadODPSTable(
         return getattr(self, "partition_spec", None)
 
     def get_columns(self):
-        return self.columns
+        return self.columns or list(self.dtypes.index)
 
     def set_pruned_columns(self, columns, *, keep_order=None):  # pragma: no cover
         self.columns = columns

maxframe/dataframe/datasource/tests/test_datasource.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 from collections import OrderedDict
 
 import numpy as np
@@ -33,6 +34,7 @@ from ..from_tensor import (
 )
 from ..index import from_pandas as from_pandas_index
 from ..index import from_tileable
+from ..read_odps_query import ColumnSchema, _resolve_task_sector
 from ..series import from_pandas as from_pandas_series
 
 ray = lazy_import("ray")
@@ -228,6 +230,7 @@ def test_from_odps_table():
     assert df.op.table_name == test_table.full_table_name
     assert df.index_value.name is None
     assert isinstance(df.index_value.value, IndexValue.RangeIndex)
+    assert df.op.get_columns() == ["col1", "col2", "col3"]
     pd.testing.assert_series_equal(
         df.dtypes,
         pd.Series(
@@ -247,6 +250,7 @@ def test_from_odps_table():
     assert df.op.table_name == test_table.full_table_name
     assert df.index_value.name is None
     assert isinstance(df.index_value.value, IndexValue.RangeIndex)
+    assert df.op.get_columns() == ["col1", "col2"]
     pd.testing.assert_series_equal(
         df.dtypes,
         pd.Series([np.dtype("O"), np.dtype("int64")], index=["col1", "col2"]),
@@ -257,6 +261,7 @@ def test_from_odps_table():
     assert df.index_value.name == "col1"
     assert isinstance(df.index_value.value, IndexValue.Index)
     assert df.index.dtype == np.dtype("O")
+    assert df.op.get_columns() == ["col2", "col3"]
     pd.testing.assert_series_equal(
         df.dtypes,
         pd.Series([np.dtype("int64"), np.dtype("float64")], index=["col2", "col3"]),
@@ -267,6 +272,7 @@ def test_from_odps_table():
 
     df = read_odps_table(test_parted_table, append_partitions=True)
     assert df.op.append_partitions is True
+    assert df.op.get_columns() == ["col1", "col2", "col3", "pt"]
     pd.testing.assert_series_equal(
         df.dtypes,
         pd.Series(
@@ -280,6 +286,7 @@ def test_from_odps_table():
     )
     assert df.op.append_partitions is True
     assert df.op.partitions == ["pt=20240103"]
+    assert df.op.get_columns() == ["col1", "col2", "pt"]
     pd.testing.assert_series_equal(
         df.dtypes,
         pd.Series(
@@ -377,3 +384,18 @@ def test_date_range():
     assert dr.index_value.is_unique == expected.is_unique
     assert dr.index_value.is_monotonic_increasing == expected.is_monotonic_increasing
     assert dr.name == expected.name
+
+
+def test_resolve_task_sector():
+    input_path = os.path.join(os.path.dirname(__file__), "test-data", "task-input.txt")
+    with open(input_path, "r") as f:
+        sector = f.read()
+    actual_sector = _resolve_task_sector("job0", sector)
+
+    assert actual_sector.job_name == "job0"
+    assert actual_sector.task_name == "M1"
+    assert actual_sector.output_target == "Screen"
+    assert len(actual_sector.schema) == 78
+    assert actual_sector.schema[0] == ColumnSchema("unnamed: 0", "bigint", "")
+    assert actual_sector.schema[1] == ColumnSchema("id", "bigint", "id_alias")
+    assert actual_sector.schema[2] == ColumnSchema("listing_url", "string", "")

maxframe/dataframe/groupby/__init__.py

@@ -14,6 +14,7 @@
 
 # noinspection PyUnresolvedReferences
 from ..core import DataFrameGroupBy, GroupBy, SeriesGroupBy
+from .core import NamedAgg
 
 
 def _install():

maxframe/dataframe/groupby/core.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from collections import namedtuple
+
 import pandas as pd
 
 from ... import opcodes
@@ -30,6 +32,9 @@ _GROUP_KEYS_NO_DEFAULT = pd_release_version >= (1, 5, 0)
 _default_group_keys = no_default if _GROUP_KEYS_NO_DEFAULT else True
 
 
+NamedAgg = namedtuple("NamedAgg", ["column", "aggfunc"])
+
+
 class DataFrameGroupByOperator(MapReduceOperator, DataFrameOperatorMixin):
     _op_type_ = opcodes.GROUPBY
 

maxframe/lib/aio/isolation.py

@@ -14,11 +14,14 @@
 
 import asyncio
 import atexit
+import itertools
 import threading
 from typing import Dict, Optional
 
 
 class Isolation:
+    _counter = itertools.count().__next__
+
     loop: asyncio.AbstractEventLoop
     _stopped: Optional[asyncio.Event]
     _thread: Optional[threading.Thread]
@@ -38,7 +41,9 @@ class Isolation:
 
     def start(self):
         if self._threaded:
-            self._thread = thread = threading.Thread(target=self._run)
+            self._thread = thread = threading.Thread(
+                name=f"IsolationThread-{self._counter()}", target=self._run
+            )
             thread.daemon = True
             thread.start()
             self._thread_ident = thread.ident

maxframe/protocol.py

@@ -46,6 +46,8 @@ BodyType = TypeVar("BodyType", bound="Serializable")
 
 
 class JsonSerializable(Serializable):
+    _ignore_non_existing_keys = True
+
     @classmethod
     def from_json(cls, serialized: dict) -> "JsonSerializable":
         raise NotImplementedError
@@ -245,6 +247,8 @@ class DagInfo(JsonSerializable):
         default_factory=dict,
     )
     error_info: Optional[ErrorInfo] = ReferenceField("error_info", default=None)
+    start_timestamp: Optional[float] = Float64Field("start_timestamp", default=None)
+    end_timestamp: Optional[float] = Float64Field("end_timestamp", default=None)
 
     @classmethod
     def from_json(cls, serialized: dict) -> "DagInfo":
@@ -265,7 +269,10 @@ class DagInfo(JsonSerializable):
             "dag_id": self.dag_id,
             "status": self.status.value,
             "progress": self.progress,
+            "start_timestamp": self.start_timestamp,
+            "end_timestamp": self.end_timestamp,
         }
+        ret = {k: v for k, v in ret.items() if v is not None}
         if self.tileable_to_result_infos:
             ret["tileable_to_result_infos"] = {
                 k: v.to_json() for k, v in self.tileable_to_result_infos.items()

maxframe/serialization/serializables/core.py

@@ -112,6 +112,7 @@ class Serializable(metaclass=SerializableMeta):
     __slots__ = ("__weakref__",)
 
     _cache_primitive_serial = False
+    _ignore_non_existing_keys = False
 
     _FIELDS: Dict[str, Field]
     _FIELD_ORDER: List[str]
@@ -128,7 +129,11 @@ class Serializable(metaclass=SerializableMeta):
         else:
             values = kwargs
         for k, v in values.items():
-            fields[k].set(self, v)
+            try:
+                fields[k].set(self, v)
+            except KeyError:
+                if not self._ignore_non_existing_keys:
+                    raise
 
     def __on_deserialize__(self):
         pass

maxframe/serialization/serializables/field.py

@@ -507,12 +507,14 @@ class ReferenceField(Field):
         tag: str,
         reference_type: Union[str, Type] = None,
         default: Any = no_default,
+        default_factory: Optional[Callable] = None,
         on_serialize: Callable[[Any], Any] = None,
         on_deserialize: Callable[[Any], Any] = None,
     ):
         super().__init__(
             tag,
             default=default,
+            default_factory=default_factory,
             on_serialize=on_serialize,
             on_deserialize=on_deserialize,
         )

maxframe/tensor/core.py

@@ -43,7 +43,7 @@ from ..serialization.serializables import (
     StringField,
     TupleField,
 )
-from ..utils import on_deserialize_shape, on_serialize_shape
+from ..utils import on_deserialize_shape, on_serialize_shape, skip_na_call
 from .utils import fetch_corner_data, get_chunk_slices
 
 logger = logging.getLogger(__name__)
@@ -181,8 +181,8 @@ class TensorData(HasShapeTileableData, _ExecuteAndFetchMixin):
     _chunks = ListField(
         "chunks",
         FieldTypes.reference(TensorChunkData),
-        on_serialize=lambda x: [it.data for it in x] if x is not None else x,
-        on_deserialize=lambda x: [TensorChunk(it) for it in x] if x is not None else x,
+        on_serialize=skip_na_call(lambda x: [it.data for it in x]),
+        on_deserialize=skip_na_call(lambda x: [TensorChunk(it) for it in x]),
     )
 
     def __init__(

maxframe/tests/test_codegen.py

@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+from typing import List, Tuple
+
+# 使用pytest生成单元测试
+import pytest
+
+from maxframe.codegen import UserCodeMixin
+from maxframe.lib import wrapped_pickle
+from maxframe.serialization.core import PickleContainer
+
+
+@pytest.mark.parametrize(
+    "input_obj, expected_output",
+    [
+        (None, "None"),
+        (10, "10"),
+        (3.14, "3.14"),
+        (True, "True"),
+        (False, "False"),
+        (b"hello", "base64.b64decode(b'aGVsbG8=')"),
+        ("hello", "'hello'"),
+        ([1, 2, 3], "[1, 2, 3]"),
+        ({"a": 1, "b": 2}, "{'a': 1, 'b': 2}"),
+        ((1, 2, 3), "(1, 2, 3)"),
+        ((1,), "(1,)"),
+        ((), "()"),
+        ({1, 2, 3}, "{1, 2, 3}"),
+        (set(), "set()"),
+    ],
+)
+def test_obj_to_python_expr(input_obj, expected_output):
+    assert UserCodeMixin.obj_to_python_expr(input_obj) == expected_output
+
+
+def test_obj_to_python_expr_custom_object():
+    class CustomClass:
+        def __init__(self, a: int, b: List[int], c: Tuple[int, int]):
+            self.a = a
+            self.b = b
+            self.c = c
+
+    custom_obj = CustomClass(1, [2, 3], (4, 5))
+    pickle_data = wrapped_pickle.dumps(custom_obj)
+    pickle_str = base64.b64encode(pickle_data)
+    custom_obj_pickle_container = PickleContainer([pickle_data])
+
+    # with class obj will not support currently
+    with pytest.raises(ValueError):
+        UserCodeMixin.obj_to_python_expr(custom_obj)
+
+    assert (
+        UserCodeMixin.obj_to_python_expr(custom_obj_pickle_container)
+        == f"cloudpickle.loads(base64.b64decode({pickle_str}), buffers=[])"
+    )

maxframe/tests/test_protocol.py

@@ -11,6 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+import json
 import time
 
 import pytest
@@ -29,28 +31,32 @@ from ..serialization import RemoteException
 from ..utils import deserialize_serializable, serialize_serializable
 
 
+def _json_round_trip(json_data: dict) -> dict:
+    return json.loads(json.dumps(json_data))
+
+
 def test_result_info_json_serialize():
-    ri = ResultInfo.from_json(ResultInfo().to_json())
+    ri = ResultInfo.from_json(_json_round_trip(ResultInfo().to_json()))
     assert type(ri) is ResultInfo
 
     ri = ODPSTableResultInfo(
         full_table_name="table_name", partition_specs=["pt=partition"]
     )
-    deserial_ri = ResultInfo.from_json(ri.to_json())
+    deserial_ri = ResultInfo.from_json(_json_round_trip(ri.to_json()))
     assert type(ri) is ODPSTableResultInfo
     assert ri.result_type == deserial_ri.result_type
     assert ri.full_table_name == deserial_ri.full_table_name
     assert ri.partition_specs == deserial_ri.partition_specs
 
     ri = ODPSTableResultInfo(full_table_name="table_name")
-    deserial_ri = ResultInfo.from_json(ri.to_json())
+    deserial_ri = ResultInfo.from_json(_json_round_trip(ri.to_json()))
     assert type(ri) is ODPSTableResultInfo
     assert ri.result_type == deserial_ri.result_type
     assert ri.full_table_name == deserial_ri.full_table_name
     assert ri.partition_specs == deserial_ri.partition_specs
 
     ri = ODPSVolumeResultInfo(volume_name="vol_name", volume_path="vol_path")
-    deserial_ri = ResultInfo.from_json(ri.to_json())
+    deserial_ri = ResultInfo.from_json(_json_round_trip(ri.to_json()))
     assert type(ri) is ODPSVolumeResultInfo
     assert ri.result_type == deserial_ri.result_type
     assert ri.volume_name == deserial_ri.volume_name
@@ -63,7 +69,7 @@ def test_error_info_json_serialize():
     except ValueError as ex:
         err_info = ErrorInfo.from_exception(ex)
 
-    deserial_err_info = ErrorInfo.from_json(err_info.to_json())
+    deserial_err_info = ErrorInfo.from_json(_json_round_trip(err_info.to_json()))
     assert deserial_err_info.error_messages == err_info.error_messages
     assert isinstance(deserial_err_info.raw_error_data, ValueError)
 
@@ -73,7 +79,7 @@ def test_error_info_json_serialize():
     with pytest.raises(RemoteException):
         mf_err_info.reraise()
 
-    deserial_err_info = ErrorInfo.from_json(mf_err_info.to_json())
+    deserial_err_info = ErrorInfo.from_json(_json_round_trip(mf_err_info.to_json()))
     assert isinstance(deserial_err_info.raw_error_data, ValueError)
     with pytest.raises(ValueError):
         deserial_err_info.reraise()
@@ -94,7 +100,9 @@ def test_dag_info_json_serialize():
         },
         error_info=err_info,
     )
-    deserial_info = DagInfo.from_json(info.to_json())
+    json_info = info.to_json()
+    json_info["non_existing_field"] = "non_existing"
+    deserial_info = DagInfo.from_json(_json_round_trip(json_info))
     assert deserial_info.session_id == info.session_id
     assert deserial_info.dag_id == info.dag_id
     assert deserial_info.status == info.status
@@ -121,7 +129,7 @@ def test_session_info_json_serialize():
         idle_timestamp=None,
         dag_infos={"test_dag_id": dag_info},
     )
-    deserial_info = SessionInfo.from_json(info.to_json())
+    deserial_info = SessionInfo.from_json(_json_round_trip(info.to_json()))
     assert deserial_info.session_id == info.session_id
     assert deserial_info.settings == info.settings
     assert deserial_info.start_timestamp == info.start_timestamp

maxframe/tests/utils.py

@@ -104,6 +104,7 @@ def run_app_in_thread(app_func):
         q = queue.Queue()
         exit_event = asyncio.Event(loop=app_loop)
         app_thread = Thread(
+            name="TestAppThread",
             target=app_thread_func,
             args=(app_loop, q, exit_event, args, kwargs),
         )

maxframe/utils.py

@@ -338,6 +338,14 @@ def deserialize_serializable(ser_serializable: bytes):
     return deserialize(header2, buffers2)
 
 
+def skip_na_call(func: Callable):
+    @functools.wraps(func)
+    def new_func(x):
+        return func(x) if x is not None else None
+
+    return new_func
+
+
 def url_path_join(*pieces):
     """Join components of url into a relative url
 
@@ -450,6 +458,9 @@ _ToThreadRetType = TypeVar("_ToThreadRetType")
 
 
 class ToThreadMixin:
+    _thread_pool_size = 1
+    _counter = itertools.count().__next__
+
     def __del__(self):
         if hasattr(self, "_pool"):
             kw = {"wait": False}
@@ -466,7 +477,10 @@ class ToThreadMixin:
         **kwargs,
     ) -> _ToThreadRetType:
         if not hasattr(self, "_pool"):
-            self._pool = concurrent.futures.ThreadPoolExecutor(1)
+            self._pool = concurrent.futures.ThreadPoolExecutor(
+                self._thread_pool_size,
+                thread_name_prefix=f"{type(self).__name__}Pool-{self._counter()}",
+            )
 
         task = asyncio.create_task(
             to_thread_pool(func, *args, **kwargs, pool=self._pool)

{maxframe-0.1.0b2.dist-info → maxframe-0.1.0b3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maxframe
-Version: 0.1.0b2
+Version: 0.1.0b3
 Summary: MaxFrame operator-based data analyze framework
 Requires-Dist: numpy >=1.19.0
 Requires-Dist: pandas >=1.0.0

{maxframe-0.1.0b2.dist-info → maxframe-0.1.0b3.dist-info}/RECORD

@@ -1,21 +1,21 @@
 maxframe/__init__.py,sha256=MgltwhBvnUQDKKmHDg9Y69TJkRGmQQ9m8-D9FC2pcLU,1007
-maxframe/_utils.cp39-win_amd64.pyd,sha256=cu_NBL6jXWstApreZwklFX4eW8vFdG5SKUpWbQ96uDQ,305664
+maxframe/_utils.cp39-win_amd64.pyd,sha256=Ig5N-JrpF21quW8LFsdBy--PdWcJwPzBLp1x6LSnp9g,305664
 maxframe/_utils.pxd,sha256=_qHN-lCY1FQgDFIrrqA79Ys0SBdonp9kXRMS93xKSYk,1187
 maxframe/_utils.pyx,sha256=_3p6aJEJ6WZYLcNZ6o4DxoxsxqadTlJXFlgDeFPxqUQ,17564
-maxframe/codegen.py,sha256=LzOBM2H5U24Eis3NA9tiy5dNe5xKsk7YaUizZKkkG44,14020
+maxframe/codegen.py,sha256=S23hTTu2fIJTv5PMIItE_NPUnadUyoiMmb-1YAqbaWw,16329
 maxframe/conftest.py,sha256=JE9I-5mP4u-vgUqYL22mNY3tqpGofM8VMe8c8VUYkzk,4403
 maxframe/env.py,sha256=xY4wjMWIJ4qLsFAQ5F-X5CrVR7dDSWiryPXni0YSK5c,1435
 maxframe/errors.py,sha256=xBnvoJjjNcHVLhwj77Dux9ut8isGVmmJXFqefmmx8Ak,711
 maxframe/extension.py,sha256=o7yiS99LWTtLF7ZX6F78UUJAqUyd-LllOXA2l69np50,2455
 maxframe/mixin.py,sha256=QfX0KqVIWDlVDSFs0lwdzLexw7lS7W_IUuK7aY1Ib8c,3624
 maxframe/opcodes.py,sha256=L-BvSFEUOMrtGJFXIH6zk2Xv_-te5VZxv5eDDCNNt0U,10566
-maxframe/protocol.py,sha256=4JOFt8zU56d7BltDxksmuQaJPSG_f18Js4XS24SzPDo,14230
+maxframe/protocol.py,sha256=N4i0ggLY131gwnxOrCgKeZwzhLKSRB171cx1lWRvUcw,14605
 maxframe/session.py,sha256=Mme-jB2hioJC6ttrXfX4XSeHLCMYoFR0ikmmhx82-vc,36624
 maxframe/typing_.py,sha256=pAgOhHHSM376N7PJLtNXvS5LHNYywz5dIjnA_hHRWSM,1133
 maxframe/udf.py,sha256=EFAAV2c8SpWKcF9_8Pocpjc4bXsEASf57Qy_Q30YH4Q,2315
-maxframe/utils.py,sha256=xbXViFjhzlAjvg_K7kHc0pxt2pSJJ_iSBlmCyXOWNAs,34644
+maxframe/utils.py,sha256=CpA4Cqf5Lg7LMKJeJFsmybqVutcSh3sSqMhdHnIi0h4,35017
 maxframe/config/__init__.py,sha256=AHo3deaCm1JnbbRX_udboJEDYrYytdvivp9RFxJcumI,671
-maxframe/config/config.py,sha256=PYJgzQy3rjVDbLzfyn9vtU4Lf7cgcJgqO5qDmNVnMUc,13321
+maxframe/config/config.py,sha256=Top-9hUEkPvaTlAGrP9GzGjwvZk6YPuHtwy4TEryKqg,13475
 maxframe/config/validators.py,sha256=pKnloh2kEOBRSsT8ks-zL8XVSaMMVIEvHvwNJlideeo,1672
 maxframe/config/tests/__init__.py,sha256=29eM5D4knhYwe3TF42naTuC5b4Ym3VeH4rK8KpdLWNY,609
 maxframe/config/tests/test_config.py,sha256=FWQZ6KBUG_jY1-KaR-GKXl7khhlTbuLlk3uaEV8koM8,2839
@@ -26,14 +26,14 @@ maxframe/core/mode.py,sha256=a-2AjLrIaqemN3pZPFhdfrPYXR7ryCLcsT1KJwWKPb0,3107
 maxframe/core/entity/__init__.py,sha256=tD4zo3KXpzLrQraHnIXeO1Q961lSsIqpbAGRK2WijVE,1336
 maxframe/core/entity/chunks.py,sha256=zKk8Iyc3IkakIDW1bMYq_zZNLrR4ZMdXH-mBuOiFerM,2202
 maxframe/core/entity/core.py,sha256=aFwjNMhTJ4ybr1WzmMVSTG211fzutzaATs14QoNh-JM,4170
-maxframe/core/entity/executable.py,sha256=Pt-ad4u2jqb9mCbOHp0DcL1Oo6kjUzF411ZxQrouqys,11239
+maxframe/core/entity/executable.py,sha256=CKxFGvFPfY_8JBprhpyndhTSLgVLtUG4G5n7Dw0dHnw,11275
 maxframe/core/entity/fuse.py,sha256=X1lI0WXj5t0flgGI5-qlVl5LoYkAdLJHk2Vv767C9G4,2350
-maxframe/core/entity/objects.py,sha256=GST_N6nwNxUv1wXaPBWty7n8Asn2hX8wfIDjrctkZP8,3119
+maxframe/core/entity/objects.py,sha256=Ys_l6cBp0HwgRmXuqYo4HsnjdbfUW4mgvek5W0IMmXY,3134
 maxframe/core/entity/output_types.py,sha256=NnNeDBVAEhD8dtPBWzpM7n6s8neVFrahjd0zMGWroCc,2735
 maxframe/core/entity/tileables.py,sha256=6jJyFscvb8sH5K_k2VaNGeUm8YrpevCtou3WSUl4Dw8,13973
 maxframe/core/entity/utils.py,sha256=454RYVbTMVW_8KnfDqUPec4kz1p98izVTC2OrzhOkao,966
 maxframe/core/graph/__init__.py,sha256=n1WiszgVu0VdXsk12oiAyggduNwu-1-9YKnfZqvmmXk,838
-maxframe/core/graph/core.cp39-win_amd64.pyd,sha256=5K_iDjXsk36IBCH6WyYsXFZYwN4NnsAt77AffVNtkpI,249344
+maxframe/core/graph/core.cp39-win_amd64.pyd,sha256=ihFU0cpiDugw4-EfUAzPbk46z6Rutez4lL37kzEi8oQ,249344
 maxframe/core/graph/core.pyx,sha256=WYlYtXXSs72vfhf2ttJO-4u85exYzy2J9mlALHOMqoA,16354
 maxframe/core/graph/entity.py,sha256=RT_xbP5niUN5D6gqZ5Pg1vUegHn8bqPk8G8A30quOVA,5730
 maxframe/core/graph/builder/__init__.py,sha256=vTRY5xRPOMHUsK0jAtNIb1BjSPGqi_6lv86AroiiiL4,718
@@ -54,7 +54,7 @@ maxframe/core/operator/tests/__init__.py,sha256=29eM5D4knhYwe3TF42naTuC5b4Ym3VeH
 maxframe/core/operator/tests/test_core.py,sha256=iqZk4AWubFLO24V_VeV6SEy5xrzBFLP9qKK6tKO0SGs,1755
 maxframe/core/tests/__init__.py,sha256=29eM5D4knhYwe3TF42naTuC5b4Ym3VeH4rK8KpdLWNY,609
 maxframe/core/tests/test_mode.py,sha256=fyRH-ksa6MogEs6kNhtXhCZyvhYqflgaXJYI3nSo-ps,2507
-maxframe/dataframe/__init__.py,sha256=tDpskfQDhuP5IoaqPFPvuzZCYW_FXAdmnD48CDQ_ysQ,2166
+maxframe/dataframe/__init__.py,sha256=t_Fnu2MNCgOr_N7mMqGU3LeZXvBF1yOHtLb5u1PoHt0,2187
 maxframe/dataframe/arrays.py,sha256=rOvhxMQars9E3SOYSu0ygBuuRVY0QV6xzengnMqKs4s,29616
 maxframe/dataframe/core.py,sha256=63TqgmOCTr5wtwbILJ6bLtHZQ6xnZd7LvVrMqR27GDQ,76034
 maxframe/dataframe/initializer.py,sha256=WW96yQjquofNFt6RPZvgWW4SBmH0OEDj8-BxpuyKThY,10552
@@ -119,12 +119,12 @@ maxframe/dataframe/datasource/from_records.py,sha256=ygpKOMXZnDdWzGxMxQ4KdGv-tJF
 maxframe/dataframe/datasource/from_tensor.py,sha256=mShHYi0fZcG7ZShFVgIezaphh8tSFqR9-nQMm5YKIhw,15146
 maxframe/dataframe/datasource/index.py,sha256=X_NShW67nYJGxaWp3qOrvyInNkz9L-XHjbApU4fHoes,4518
 maxframe/dataframe/datasource/read_csv.py,sha256=IvQihmpcZIdzSD7ziX92aTAHNyP5WnTgd2cZz_h43sQ,24668
-maxframe/dataframe/datasource/read_odps_query.py,sha256=BbIfSc1dOzauZQYLAIxKO-Z2VGZmBQ884UqSdalmG_o,10115
-maxframe/dataframe/datasource/read_odps_table.py,sha256=kZDpJvq3d39hLYa4oznk2LjJXipOUkfXadDhxPFSZbs,9267
+maxframe/dataframe/datasource/read_odps_query.py,sha256=QKU_7R6dd6GXzNX4IN9uwu3jB17MfhTr2nUtx0LFSNk,10116
+maxframe/dataframe/datasource/read_odps_table.py,sha256=ocSKQQ7SwIkzliFCFWNzy3e8J3CBZsf4971oRdlgdks,9294
 maxframe/dataframe/datasource/read_parquet.py,sha256=SZPrWoax2mwMBNvRk_3lkS72pZLe-_X_GwQ1JROBMs4,14952
 maxframe/dataframe/datasource/series.py,sha256=elQVupKETh-hUHI2fTu8TRxBE729Vyrmpjx17XlRV-8,1964
 maxframe/dataframe/datasource/tests/__init__.py,sha256=29eM5D4knhYwe3TF42naTuC5b4Ym3VeH4rK8KpdLWNY,609
-maxframe/dataframe/datasource/tests/test_datasource.py,sha256=mJc8z0dSDyT1nni-VTij6LLcu_sp8Cj0eekxga4Ritk,13999
+maxframe/dataframe/datasource/tests/test_datasource.py,sha256=UumRBjE-bIuCi7Z4_3t8qb58ZcF8ePRZf3xF7DTvqIA,15041
 maxframe/dataframe/datastore/__init__.py,sha256=MmlHYvFacMReOHDQMXF-z2bCsLyrSHYBVwIlCsZGOK4,810
 maxframe/dataframe/datastore/to_csv.py,sha256=lheaF3ZmBPrcwcWyhK5gEVAGIaLJbvTyVAzqZFGG7eM,8026
 maxframe/dataframe/datastore/to_odps.py,sha256=Ml_iF9AspqIgGbeOAjTF3ukAwv-7SizribuqXZdxuXo,5776
@@ -135,10 +135,10 @@ maxframe/dataframe/extensions/tests/__init__.py,sha256=29eM5D4knhYwe3TF42naTuC5b
 maxframe/dataframe/extensions/tests/test_extensions.py,sha256=oDnVwQx-o8p4wmen8ZS3gnudOAihFAdNKQhSCqNNXzQ,1324
 maxframe/dataframe/fetch/__init__.py,sha256=W1arTCAjD0v_bdVeIzpJMnil3h0Ucsn29zFWmgZcYck,668
 maxframe/dataframe/fetch/core.py,sha256=27VANjpMm2rdCg1KPZxWZrKWNuNgSMzZrordI05mqWc,3424
-maxframe/dataframe/groupby/__init__.py,sha256=YjjKMh841TLcU-gik7n9fjeYNFwtYjvrHNq4zf83UBM,3415
+maxframe/dataframe/groupby/__init__.py,sha256=wMjmvk4ced1uCm7bw0oodIKvaep61KhupriL9JRRq5w,3443
 maxframe/dataframe/groupby/aggregation.py,sha256=cUnu-Bj6YD1TVkaafwL2aGIIqixLEq7s9-7BQ_1T2DI,12303
 maxframe/dataframe/groupby/apply.py,sha256=DQHyEfqj-3tfK-CxwpdVgya0_YC9dImeWYPZJDw7ckk,9735
-maxframe/dataframe/groupby/core.py,sha256=BPcDrMN710phqwV1_F7kZKKan7ZqxoER-Jd4mXxMMyw,6154
+maxframe/dataframe/groupby/core.py,sha256=NG6e3sqIu5dnBw9_DCQEDtsnxM5e4Yl1oD7Z_qjdtWA,6254
 maxframe/dataframe/groupby/cum.py,sha256=A7vIWLsb50VLu3yAngO-BfZecjWj0Fk6TZ5v4uQEAPM,3879
 maxframe/dataframe/groupby/fill.py,sha256=AXRmA_j-m7ig0udLCJ02FwIce2GLQ2U8KlnuCe-NY3U,4862
 maxframe/dataframe/groupby/getitem.py,sha256=owNzoE8UEfM41dfuntKkRBjjYYbY8O8CMJchIhCEyds,3344
@@ -283,7 +283,7 @@ maxframe/learn/contrib/pytorch/tests/test_pytorch.py,sha256=GHP-oD5uMU8LD90Jt2cH
 maxframe/lib/__init__.py,sha256=_PB28W40qku6YiT8fJYqdmEdRMQfelOwGeksCOZJfCc,657
 maxframe/lib/compression.py,sha256=QQpNK79iUC9zck74I0HKMhapSRnLBXtTRyS91taEVIc,1497
 maxframe/lib/functools_compat.py,sha256=2LTrkSw5i-z5E9XCtZzfg9-0vPrYxicKvDjnnNrAL1Q,2697
-maxframe/lib/mmh3.cp39-win_amd64.pyd,sha256=2cgKxEhMe8AxmIhYaKwe2c3mTz7dLWQ4vHPTAzGEEA4,17408
+maxframe/lib/mmh3.cp39-win_amd64.pyd,sha256=sy3Uoiv0kFIwa-3jA-x5HvIpHgJdrcQi6zOKIBPP31A,17408
 maxframe/lib/version.py,sha256=VOVZu3KHS53YUsb_vQsT7AyHwcCWAgc-3bBqV5ANcbQ,18941
 maxframe/lib/wrapped_pickle.py,sha256=bzEaokhAZlkjXqw1xfeKO1KX2awhKIz_1RT81yPPoag,3949
 maxframe/lib/aio/__init__.py,sha256=xzIYnV42_7CYuDTTv8svscIXQeJMF0nn8AXMbpv173M,963
@@ -291,7 +291,7 @@ maxframe/lib/aio/_runners.py,sha256=zhDC92KxrYxLEufo5Hk8QU-mTVOxNL7IM9pZXas_nDg,
 maxframe/lib/aio/_threads.py,sha256=cDaEKg5STncq9QTPUUwehJ722vgueqBoB1C-NeoHN8E,1363
 maxframe/lib/aio/base.py,sha256=Ol0MnkcsBRfsQdZWceYfaWVtNOuiHzY8EYo2Zh0QFvM,2240
 maxframe/lib/aio/file.py,sha256=uy2LM_U8-Snpf45yZqUQRR_0hZT5UXZnwq0qENpMI6k,2097
-maxframe/lib/aio/isolation.py,sha256=zUqytRXUHRniUrZCRtsJkBldP-RnGWv3KpelQ3hyRdU,2723
+maxframe/lib/aio/isolation.py,sha256=wIliQ1qFtGV_cZ4stE21QHKUVtq7j88m7ZuZF8Ye2iE,2861
 maxframe/lib/aio/lru.py,sha256=hZ0QY8VWhZr06B11YqjEONKcjySP7oKaa-p9evwnxZY,7133
 maxframe/lib/aio/parallelism.py,sha256=Q3dXir6wr5vG2SmVSz0n6BdH7d5mhMTohfeFs5JDTtU,1272
 maxframe/lib/aio/tests/__init__.py,sha256=29eM5D4knhYwe3TF42naTuC5b4Ym3VeH4rK8KpdLWNY,609
@@ -346,7 +346,7 @@ maxframe/remote/core.py,sha256=w_eTDEs0O7iIzLn1YrMGh2gcNAzzbqV0mx2bRT7su_U,7001
 maxframe/remote/run_script.py,sha256=k93-vaFLUanWoBRai4-78DX_SLeZ8_rbbxcCtOIXZO8,3677
 maxframe/serialization/__init__.py,sha256=nxxU7CI6MRcL3sjA1KmLkpTGKA3KG30FKl-MJJ0MCdI,947
 maxframe/serialization/arrow.py,sha256=OMeDjLcPgagqzokG7g3Vhwm6Xw1j-Kph1V2QsIwi6dw,3513
-maxframe/serialization/core.cp39-win_amd64.pyd,sha256=1do04vxT1TRzI-yR9cIIxoI3SFQ0V1ETTRKxHue-Rfw,393216
+maxframe/serialization/core.cp39-win_amd64.pyd,sha256=v5hTr21JOIqdmz-tR5jJ5XFr-bymjyOxBdiUKjge25w,393216
 maxframe/serialization/core.pxd,sha256=Fymih3Wo-CrOY27_o_DRINdbRGR7mgiT-XCaXCXafxM,1347
 maxframe/serialization/core.pyx,sha256=Qmipu3LiJGIBVy_7d4tSJqcYWnG5xj2I7IaPv2PSq5E,35078
 maxframe/serialization/exception.py,sha256=e7bZyPlZ8XhSCdeOwlYreq0HazPXKOgOA6r9Q4Ecn2Y,3113
@@ -355,8 +355,8 @@ maxframe/serialization/numpy.py,sha256=ENrFKl24mtYyO1vZRLwHvMD0r4z_UI7J2-yNlmfWS
 maxframe/serialization/pandas.py,sha256=3aPzDOg9UYSI9GFpWm2aJc8EAi-d-timM8vQ8kTL3Cg,7349
 maxframe/serialization/scipy.py,sha256=fGwQ5ZreymrMT8g7TneATfFdKFF7YPNZQqgWgMa3J8M,2498
 maxframe/serialization/serializables/__init__.py,sha256=rlQhIaSAVzz4KYkc5shEHFZDPd6WDMPkxalU76yjJ3M,1406
-maxframe/serialization/serializables/core.py,sha256=9QlrQIRWSf0Ocs3YB4_n62v03RAkrb1rGN8AFCuzumI,9053
-maxframe/serialization/serializables/field.py,sha256=Rv9qFKCE0G9TSJXo1VU8mYu_OLakFsiZHC0enDrKIBs,16540
+maxframe/serialization/serializables/core.py,sha256=QUjHQPG_Qd5yh_bW-mCdapY5uKUl-s1axrM2N3eomyQ,9227
+maxframe/serialization/serializables/field.py,sha256=DVott3HAbne4UvN-heSFS9gSl0wCxV5RssS738FCjzk,16639
 maxframe/serialization/serializables/field_type.py,sha256=hkxrXT2SL_tATuobtJDfL4DzzVP2hJjDlC3PrJg6ZKo,15454
 maxframe/serialization/serializables/tests/__init__.py,sha256=29eM5D4knhYwe3TF42naTuC5b4Ym3VeH4rK8KpdLWNY,609
 maxframe/serialization/serializables/tests/test_field_type.py,sha256=uG87-bdG8xGmjrubEHCww1ZKmRupSvnNKnZoV2SnwYM,4502
@@ -365,7 +365,7 @@ maxframe/serialization/tests/__init__.py,sha256=29eM5D4knhYwe3TF42naTuC5b4Ym3VeH
 maxframe/serialization/tests/test_serial.py,sha256=9G2CbPBHINwcZ038pRwZON_OtH-JVXZ8w66BLYWP578,12923
 maxframe/tensor/__init__.py,sha256=aomZCK-bt5OYyRLGcbACxtFLrWIp14F4R3P79zwbN5E,3694
 maxframe/tensor/array_utils.py,sha256=xr_Ng-4dETJFjsMfWi5gbTPM9mRmPvRWj8QY2WKjmCg,5129
-maxframe/tensor/core.py,sha256=u9HPbYj1gOF4m7WJpjSCprIHR-3HEEwK_LhjPNmro7g,22644
+maxframe/tensor/core.py,sha256=-G-UzY81GTKj2SD9FQLqBg-UDod5LjjrEA-uF16ofms,22638
 maxframe/tensor/operators.py,sha256=8VsSZ8OcImGkSRQvrYlV05KMHGsroAYmW1o9RM2yV1U,3584
 maxframe/tensor/utils.py,sha256=An35s6MrbltYvN8WYzjKCjyozTDbGQrvUW_qz8KnA94,23632
 maxframe/tensor/arithmetic/__init__.py,sha256=SUlcG0Mf9ddgxAdydenuJ9eY5yVu0TgKfpBujI3OX4w,9695
@@ -603,26 +603,27 @@ maxframe/tensor/statistics/quantile.py,sha256=HrPxQoRXTEGf-5m79osqhUoSTFpWmIwmhO
 maxframe/tensor/ufunc/__init__.py,sha256=8QUi-cPvvbsD7i7LOeZ9sc0v1XXd7lt-XV5pQKbVZJs,821
 maxframe/tensor/ufunc/ufunc.py,sha256=XRtGlhdrW7H--mrc8fTBOlUP0mzKpd9tdRtCuLDymtc,7383
 maxframe/tests/__init__.py,sha256=29eM5D4knhYwe3TF42naTuC5b4Ym3VeH4rK8KpdLWNY,609
-maxframe/tests/test_protocol.py,sha256=n7LSC42JtGQuCRt0XY8sd9ewf6i_1C0LE3ux9yNMGp8,4993
+maxframe/tests/test_codegen.py,sha256=h3TKqP4zghxTn1twH7gR9jOe6NKXdCC1B4u0chlUrpY,2277
+maxframe/tests/test_protocol.py,sha256=IgZT2CBH1dv6V1DwSq-PjvrUhvtOf8Mab6dnWhAT3No,5331
 maxframe/tests/test_utils.py,sha256=0Iey3O6zrGI1yQU2OSpWavJNvhUjrmdkct4-27tkGUM,12353
-maxframe/tests/utils.py,sha256=ulCRwCjaJeoMo4NwvekJq5BhEJzzr9sdTn-cOx9HG_A,4697
+maxframe/tests/utils.py,sha256=gCre-8BApU4-AEun9WShm4Ff5a9a_oKxvLNneESXBjU,4732
 maxframe_client/__init__.py,sha256=xqlN69LjvAp2bNCaT9d82U9AF5WKi_c4UOheEW1wV9E,741
 maxframe_client/conftest.py,sha256=UWWMYjmohHL13hLl4adb0gZPLRdBVOYVvsFo6VZruI0,658
 maxframe_client/fetcher.py,sha256=ajt14PYVEXKShXSx-qTL_8adMCXBEsZoOhHr3yx95-M,7015
 maxframe_client/clients/__init__.py,sha256=29eM5D4knhYwe3TF42naTuC5b4Ym3VeH4rK8KpdLWNY,609
-maxframe_client/clients/framedriver.py,sha256=_Hv_pcV0Rmc3v8sNLzrZyTYz5AyoavDPOs8jtotFQWU,4570
+maxframe_client/clients/framedriver.py,sha256=upN6C1eZrCpLTsS6fihWOMy392psWfo0bw2XgSLI_Yg,4581
 maxframe_client/clients/spe.py,sha256=uizNBejhU_FrMhsgsFgDnq7gL7Cxk803LeLYmr3nmxs,3697
 maxframe_client/session/__init__.py,sha256=9zFCd3zkSADESAFc4SPoQ2nkvRwsIhhpNNO2TtSaWbU,854
 maxframe_client/session/consts.py,sha256=nD-D0zHXumbQI8w3aUyltJS59K5ftipf3xCtHNLmtc8,1380
 maxframe_client/session/graph.py,sha256=GSZaJ-PV4DK8bTcNtoSoY5kDTyyIRAKleh4tOCSUbsI,4470
 maxframe_client/session/odps.py,sha256=RG7_28UaS_8tgJUOa4ohw2QtwX2fF4yqsGGy2MksQWI,16700
-maxframe_client/session/task.py,sha256=pJYSdOo-nTXTz7An0u2vZop1MtQKMhAAhiT2m8ilk38,11142
+maxframe_client/session/task.py,sha256=R8x8OERIb673vTq-o0ig6Zy2NT4_jvi8AbLhyMaljo8,11409
 maxframe_client/session/tests/__init__.py,sha256=29eM5D4knhYwe3TF42naTuC5b4Ym3VeH4rK8KpdLWNY,609
-maxframe_client/session/tests/test_task.py,sha256=lm2q1tWZiaYeU0D11_f_t7acZLLi1Ts-nlIFpZHsm4U,2684
+maxframe_client/session/tests/test_task.py,sha256=861usEURVXeTUzfJYZmBfwsHfZFexG23mMtT5IJOOm4,3364
 maxframe_client/tests/__init__.py,sha256=29eM5D4knhYwe3TF42naTuC5b4Ym3VeH4rK8KpdLWNY,609
 maxframe_client/tests/test_fetcher.py,sha256=7iYXLMIoCJLfgUkjB2HBkV-sqQ-xGlhtzfp9hRJz_kM,3605
-maxframe_client/tests/test_session.py,sha256=vTycMT6IMtPQcpVFCXFWLniHXj6m03JsSciJ7x_kAbU,6443
-maxframe-0.1.0b2.dist-info/METADATA,sha256=FLBUoJqc372CTgONZ6NrajzjXo3jFdUiVp1ZQH4HSXk,3147
-maxframe-0.1.0b2.dist-info/WHEEL,sha256=Z6c-bE0pUM47a70GvqO_SvH_XXU0lm62gEAKtoNJ08A,100
-maxframe-0.1.0b2.dist-info/top_level.txt,sha256=64x-fc2q59c_vXwNUkehyjF1vb8JWqFSdYmUqIFqoTM,31
-maxframe-0.1.0b2.dist-info/RECORD,,
+maxframe_client/tests/test_session.py,sha256=s8pxf0I6PjOd6ZZQ4IYjfOM_3F3wf6SGPRMz0tAZFmo,6514
+maxframe-0.1.0b3.dist-info/METADATA,sha256=aZIVKoKmfRSjIExk2K16jB0fJj5iApw9FhoHqv6hs_k,3147
+maxframe-0.1.0b3.dist-info/WHEEL,sha256=Z6c-bE0pUM47a70GvqO_SvH_XXU0lm62gEAKtoNJ08A,100
+maxframe-0.1.0b3.dist-info/top_level.txt,sha256=64x-fc2q59c_vXwNUkehyjF1vb8JWqFSdYmUqIFqoTM,31
+maxframe-0.1.0b3.dist-info/RECORD,,

maxframe_client/clients/framedriver.py

@@ -14,6 +14,7 @@
 
 from typing import Any, Dict, List
 
+import msgpack
 from tornado import httpclient
 
 from maxframe.core import TileableGraph
@@ -28,7 +29,6 @@ from maxframe.protocol import (
 )
 from maxframe.typing_ import TimeoutType
 from maxframe.utils import (
-    deserialize_serializable,
     format_timeout_params,
     serialize_serializable,
     wait_http_response,
@@ -47,12 +47,12 @@ class FrameDriverClient:
         resp = await httpclient.AsyncHTTPClient().fetch(
             req_url, method="POST", body=serialize_serializable(req_body)
         )
-        return deserialize_serializable(resp.body).body
+        return SessionInfo.from_json(msgpack.loads(resp.body))
 
     async def get_session(self, session_id: str) -> SessionInfo:
         req_url = f"{self._endpoint}/api/sessions/{session_id}"
         resp = await httpclient.AsyncHTTPClient().fetch(req_url, method="GET")
-        return deserialize_serializable(resp.body).body
+        return SessionInfo.from_json(msgpack.loads(resp.body))
 
     async def delete_session(self, session_id: str):
         req_url = f"{self._endpoint}/api/sessions/{session_id}"
@@ -71,12 +71,12 @@ class FrameDriverClient:
             method="POST",
             body=serialize_serializable(ProtocolBody(body=req_body)),
         )
-        return deserialize_serializable(resp.body).body
+        return DagInfo.from_json(msgpack.loads(resp.body))
 
     async def get_dag_info(self, session_id: str, dag_id: str) -> DagInfo:
         req_url = f"{self._endpoint}/api/sessions/{session_id}/dags/{dag_id}"
         resp = await httpclient.AsyncHTTPClient().fetch(req_url, method="GET")
-        return deserialize_serializable(resp.body).body
+        return DagInfo.from_json(msgpack.loads(resp.body))
 
     async def wait_dag(self, session_id: str, dag_id: str, timeout: TimeoutType = None):
         query_part = format_timeout_params(timeout)
@@ -87,7 +87,7 @@ class FrameDriverClient:
             resp = await wait_http_response(
                 req_url, method="GET", request_timeout=timeout
             )
-            info = deserialize_serializable(resp.body).body
+            info = DagInfo.from_json(msgpack.loads(resp.body))
         except TimeoutError:
             info = await self.get_dag_info(session_id, dag_id)
         return info
@@ -103,7 +103,7 @@ class FrameDriverClient:
             resp = await wait_http_response(
                 req_url, method="DELETE", request_timeout=timeout
             )
-            info = deserialize_serializable(resp.body).body
+            info = DagInfo.from_json(msgpack.loads(resp.body))
         except TimeoutError:
             info = await self.get_dag_info(session_id, dag_id)
         return info

maxframe_client/session/task.py

@@ -112,13 +112,19 @@ class MaxFrameInstanceCaller(MaxFrameServiceCaller):
         odps_entry: ODPS,
         task_name: Optional[str] = None,
         project: Optional[str] = None,
-        priority: Optional[str] = None,
+        priority: Optional[int] = None,
         running_cluster: Optional[str] = None,
         nested_instance_id: Optional[str] = None,
         major_version: Optional[str] = None,
         output_format: Optional[str] = None,
         **kwargs,
     ):
+        if callable(odps_options.get_priority):
+            default_priority = odps_options.get_priority(odps_entry)
+        else:
+            default_priority = odps_options.priority
+        priority = priority if priority is not None else default_priority
+
         self._odps_entry = odps_entry
         self._task_name = task_name
         self._project = project
@@ -126,6 +132,7 @@ class MaxFrameInstanceCaller(MaxFrameServiceCaller):
         self._running_cluster = running_cluster
         self._major_version = major_version
         self._output_format = output_format or MAXFRAME_OUTPUT_MSGPACK_FORMAT
+
         if nested_instance_id is None:
             self._nested = False
             self._instance = None

maxframe_client/session/tests/test_task.py

@@ -18,6 +18,7 @@ import os
 import mock
 from defusedxml import ElementTree
 from odps import ODPS
+from odps import options as odps_options
 
 from ...session.consts import MAXFRAME_OUTPUT_JSON_FORMAT
 from ...session.task import MaxFrameInstanceCaller, MaxFrameTask
@@ -27,17 +28,20 @@ expected_file_dir = os.path.join(os.path.dirname(__file__), "expected-data")
 
 def test_maxframe_instance_caller_creating_session():
     o = ODPS.from_environments()
-    task_caller = MaxFrameInstanceCaller(
-        odps_entry=o,
-        task_name="task_test",
-        major_version="test_version",
-        output_format=MAXFRAME_OUTPUT_JSON_FORMAT,
-        priority="100",
-        running_cluster="test_cluster",
-    )
+
+    def create_caller(**kwargs):
+        kw = dict(
+            odps_entry=o,
+            task_name="task_test",
+            major_version="test_version",
+            output_format=MAXFRAME_OUTPUT_JSON_FORMAT,
+            running_cluster="test_cluster",
+        )
+        kw.update(**kwargs)
+        return MaxFrameInstanceCaller(**kw)
 
     def mock_create(self, task: MaxFrameTask, priority=None, running_cluster=None):
-        assert priority == "100"
+        assert priority == 100
         assert running_cluster == "test_cluster"
         root = ElementTree.parse(
             os.path.join(expected_file_dir, "create_session.xml")
@@ -62,6 +66,20 @@ def test_maxframe_instance_caller_creating_session():
         target="maxframe_client.session.task.MaxFrameInstanceCaller",
         _wait_instance_task_ready=mock.DEFAULT,
         get_session=mock.DEFAULT,
-    ):
-        with mock.patch("odps.models.instances.BaseInstances.create", mock_create):
+    ), mock.patch("odps.models.instances.BaseInstances.create", mock_create):
+        task_caller = create_caller(priority=100)
+        task_caller.create_session()
+
+        old_priority = odps_options.priority
+        old_get_priority = odps_options.get_priority
+        try:
+            task_caller = create_caller(priority=100)
+            odps_options.priority = 100
+            task_caller.create_session()
+
+            odps_options.priority = None
+            odps_options.get_priority = lambda _: 100
             task_caller.create_session()
+        finally:
+            odps_options.priority = old_priority
+            odps_options.get_priority = old_get_priority

maxframe_client/tests/test_session.py

@@ -24,6 +24,7 @@ from odps import ODPS
 import maxframe.dataframe as md
 import maxframe.remote as mr
 from maxframe.core import ExecutableTuple, TileableGraph
+from maxframe.lib.aio import stop_isolation
 from maxframe.protocol import ResultInfo
 from maxframe.serialization import RemoteException
 from maxframe.session import new_session
@@ -52,6 +53,7 @@ def start_mock_session(framedriver_app):  # noqa: F811
             time.sleep(5)  # Wait for temp table deleted
         else:
             session.reset_default()
+        stop_isolation()
 
 
 def test_simple_run_dataframe(start_mock_session):