maxframe 0.1.0b2__tar.gz → 0.1.0b3__tar.gz

{maxframe-0.1.0b2 → maxframe-0.1.0b3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maxframe
-Version: 0.1.0b2
+Version: 0.1.0b3
 Summary: MaxFrame operator-based data analyze framework
 Provides-Extra: dev
 Provides-Extra: test

{maxframe-0.1.0b2 → maxframe-0.1.0b3}/maxframe/codegen.py

@@ -17,7 +17,7 @@ import base64
 import dataclasses
 import logging
 from enum import Enum
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union
 
 from odps.types import OdpsSchema
 from odps.utils import camel_to_underline
@@ -30,6 +30,7 @@ from .odpsio import build_dataframe_table_meta
 from .odpsio.schema import pandas_to_odps_schema
 from .protocol import DataFrameTableMeta, ResultInfo
 from .serialization import PickleContainer
+from .serialization.serializables import Serializable, StringField
 from .typing_ import PandasObjectTypes
 from .udf import MarkedFunction
 
@@ -48,8 +49,11 @@ class CodeGenResult:
     constants: Dict[str, Any]
 
 
-class AbstractUDF(abc.ABC):
-    _session_id: str
+class AbstractUDF(Serializable):
+    _session_id: str = StringField("session_id")
+
+    def __init__(self, session_id: Optional[str] = None, **kw):
+        super().__init__(_session_id=session_id, **kw)
 
     @property
     def name(self) -> str:
@@ -74,7 +78,66 @@ class AbstractUDF(abc.ABC):
 
 class UserCodeMixin:
     @classmethod
-    def generate_pickled_codes(cls, code_to_pickle: Any) -> List[str]:
+    def obj_to_python_expr(cls, obj: Any = None) -> str:
+        """
+        Parameters
+        ----------
+        obj
+            The object to convert to python expr.
+        Returns
+        -------
+        str :
+            The str type content equals to the object when use in the python code directly.
+        """
+        if obj is None:
+            return "None"
+
+        if isinstance(obj, (int, float)):
+            return repr(obj)
+
+        if isinstance(obj, bool):
+            return "True" if obj else "False"
+
+        if isinstance(obj, bytes):
+            base64_bytes = base64.b64encode(obj)
+            return f"base64.b64decode({base64_bytes})"
+
+        if isinstance(obj, str):
+            return repr(obj)
+
+        if isinstance(obj, list):
+            return (
+                f"[{', '.join([cls.obj_to_python_expr(element) for element in obj])}]"
+            )
+
+        if isinstance(obj, dict):
+            items = (
+                f"{repr(key)}: {cls.obj_to_python_expr(value)}"
+                for key, value in obj.items()
+            )
+            return f"{{{', '.join(items)}}}"
+
+        if isinstance(obj, tuple):
+            return f"({', '.join([cls.obj_to_python_expr(sub_obj) for sub_obj in obj])}{',' if len(obj) == 1 else ''})"
+
+        if isinstance(obj, set):
+            return (
+                f"{{{', '.join([cls.obj_to_python_expr(sub_obj) for sub_obj in obj])}}}"
+                if obj
+                else "set()"
+            )
+
+        if isinstance(obj, PickleContainer):
+            return UserCodeMixin.generate_pickled_codes(obj, None)
+
+        raise ValueError(f"not support arg type {type(obj)}")
+
+    @classmethod
+    def generate_pickled_codes(
+        cls,
+        code_to_pickle: Any,
+        unpicked_data_var_name: Union[str, None] = "pickled_data",
+    ) -> str:
         """
         Generate pickled codes. The final pickled variable is called 'pickled_data'.
 
@@ -82,20 +145,20 @@ class UserCodeMixin:
         ----------
         code_to_pickle: Any
             The code to be pickled.
+        unpicked_data_var_name: str
+            The variables in code used to hold the loads object from the cloudpickle
 
         Returns
         -------
-        List[str] :
-            The code snippets of pickling, the final variable is called 'pickled_data'.
+        str :
+            The code snippets of pickling, the final variable is called 'pickled_data' by default.
         """
         pickled, buffers = cls.dump_pickled_data(code_to_pickle)
-        pickled = base64.b64encode(pickled)
-        buffers = [base64.b64encode(b) for b in buffers]
-        buffers_str = ", ".join(f"base64.b64decode(b'{b.decode()}')" for b in buffers)
-        return [
-            f"base64_data = base64.b64decode(b'{pickled.decode()}')",
-            f"pickled_data = cloudpickle.loads(base64_data, buffers=[{buffers_str}])",
-        ]
+        pickle_loads_expr = f"cloudpickle.loads({cls.obj_to_python_expr(pickled)}, buffers={cls.obj_to_python_expr(buffers)})"
+        if unpicked_data_var_name:
+            return f"{unpicked_data_var_name} = {pickle_loads_expr}"
+
+        return pickle_loads_expr
 
     @staticmethod
     def dump_pickled_data(
@@ -114,8 +177,9 @@ class UserCodeMixin:
 
 
 class BigDagCodeContext(metaclass=abc.ABCMeta):
-    def __init__(self, session_id: str = None):
+    def __init__(self, session_id: str = None, subdag_id: str = None):
         self._session_id = session_id
+        self._subdag_id = subdag_id
         self._tileable_key_to_variables = dict()
         self.constants = dict()
         self._data_table_meta_cache = dict()
@@ -142,10 +206,14 @@ class BigDagCodeContext(metaclass=abc.ABCMeta):
         except KeyError:
             var_name = self._tileable_key_to_variables[
                 tileable.key
-            ] = f"var_{self._next_var_id}"
-            self._next_var_id += 1
+            ] = self.next_var_name()
             return var_name
 
+    def next_var_name(self) -> str:
+        var_name = f"var_{self._next_var_id}"
+        self._next_var_id += 1
+        return var_name
+
     def get_odps_schema(
         self, data: PandasObjectTypes, unknown_as_string: bool = False
     ) -> OdpsSchema:
@@ -275,9 +343,10 @@ class BigDagCodeGenerator(metaclass=abc.ABCMeta):
     engine_priority: int = 0
     _extension_loaded = False
 
-    def __init__(self, session_id: str):
+    def __init__(self, session_id: str, subdag_id: str = None):
         self._session_id = session_id
-        self._context = self._init_context(session_id)
+        self._subdag_id = subdag_id
+        self._context = self._init_context(session_id, subdag_id)
 
     @classmethod
     def _load_engine_extensions(cls):
@@ -307,7 +376,7 @@ class BigDagCodeGenerator(metaclass=abc.ABCMeta):
         raise NotImplementedError
 
     @abc.abstractmethod
-    def _init_context(self, session_id: str) -> BigDagCodeContext:
+    def _init_context(self, session_id: str, subdag_id: str) -> BigDagCodeContext:
         raise NotImplementedError
 
     def _generate_comments(

{maxframe-0.1.0b2 → maxframe-0.1.0b3}/maxframe/config/config.py

@@ -340,6 +340,12 @@ default_options.register_option(
     validator=is_integer,
     remote=True,
 )
+default_options.register_option(
+    "session.subinstance_priority",
+    None,
+    validator=any_validator(is_null, is_integer),
+    remote=True,
+)
 
 default_options.register_option("warn_duplicated_execution", False, validator=is_bool)
 default_options.register_option("dataframe.use_arrow_dtype", True, validator=is_bool)

{maxframe-0.1.0b2 → maxframe-0.1.0b3}/maxframe/core/entity/executable.py

@@ -66,6 +66,7 @@ class DecrefRunner:
         if self._decref_thread:  # pragma: no branch
             self._queue.put_nowait((None, None, None))
             self._decref_thread.join(1)
+        self._decref_thread = None
 
     def put(self, key: str, session_ref: ref):
         if self._decref_thread is None:

{maxframe-0.1.0b2 → maxframe-0.1.0b3}/maxframe/core/entity/objects.py

@@ -15,6 +15,7 @@
 from typing import Any, Dict
 
 from ...serialization.serializables import FieldTypes, ListField
+from ...utils import skip_na_call
 from .chunks import Chunk, ChunkData
 from .core import Entity
 from .executable import _ToObjectMixin
@@ -62,8 +63,8 @@ class ObjectData(TileableData, _ToObjectMixin):
     _chunks = ListField(
         "chunks",
         FieldTypes.reference(ObjectChunkData),
-        on_serialize=lambda x: [it.data for it in x] if x is not None else x,
-        on_deserialize=lambda x: [ObjectChunk(it) for it in x] if x is not None else x,
+        on_serialize=skip_na_call(lambda x: [it.data for it in x]),
+        on_deserialize=skip_na_call(lambda x: [ObjectChunk(it) for it in x]),
     )
 
     def __init__(self, op=None, nsplits=None, **kw):

{maxframe-0.1.0b2 → maxframe-0.1.0b3}/maxframe/dataframe/__init__.py

@@ -39,6 +39,7 @@ from .datasource.read_odps_query import read_odps_query
 from .datasource.read_odps_table import read_odps_table
 from .datasource.read_parquet import read_parquet
 from .datastore.to_odps import to_odps_table
+from .groupby import NamedAgg
 from .initializer import DataFrame, Index, Series, read_pandas
 from .merge import concat, merge
 from .misc.cut import cut
@@ -52,7 +53,7 @@ from .reduction import CustomReduction, unique
 from .tseries.to_datetime import to_datetime
 
 try:
-    from pandas import NA, NamedAgg, Timestamp
+    from pandas import NA, Timestamp
 except ImportError:  # pragma: no cover
     pass
 

{maxframe-0.1.0b2 → maxframe-0.1.0b3}/maxframe/dataframe/datasource/read_odps_query.py

@@ -46,7 +46,7 @@ _EXPLAIN_TASK_SCHEMA_REGEX = re.compile(
     r"In Task ([^:]+)[\S\s]+FS: output: ([^\n #]+)[\s\S]+schema:\s+([\S\s]+)$",
     re.MULTILINE,
 )
-_EXPLAIN_COLUMN_REGEX = re.compile(r"([^ ]+) \(([^)]+)\)(?:| AS ([^ ]+))(?:\n|$)")
+_EXPLAIN_COLUMN_REGEX = re.compile(r"([^\(]+) \(([^)]+)\)(?:| AS ([^ ]+))(?:\n|$)")
 
 
 @dataclasses.dataclass

{maxframe-0.1.0b2 → maxframe-0.1.0b3}/maxframe/dataframe/datasource/read_odps_table.py

@@ -69,7 +69,7 @@ class DataFrameReadODPSTable(
         return getattr(self, "partition_spec", None)
 
     def get_columns(self):
-        return self.columns
+        return self.columns or list(self.dtypes.index)
 
     def set_pruned_columns(self, columns, *, keep_order=None):  # pragma: no cover
         self.columns = columns

{maxframe-0.1.0b2 → maxframe-0.1.0b3}/maxframe/dataframe/datasource/tests/test_datasource.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 from collections import OrderedDict
 
 import numpy as np
@@ -33,6 +34,7 @@ from ..from_tensor import (
 )
 from ..index import from_pandas as from_pandas_index
 from ..index import from_tileable
+from ..read_odps_query import ColumnSchema, _resolve_task_sector
 from ..series import from_pandas as from_pandas_series
 
 ray = lazy_import("ray")
@@ -228,6 +230,7 @@ def test_from_odps_table():
     assert df.op.table_name == test_table.full_table_name
     assert df.index_value.name is None
     assert isinstance(df.index_value.value, IndexValue.RangeIndex)
+    assert df.op.get_columns() == ["col1", "col2", "col3"]
     pd.testing.assert_series_equal(
         df.dtypes,
         pd.Series(
@@ -247,6 +250,7 @@ def test_from_odps_table():
     assert df.op.table_name == test_table.full_table_name
     assert df.index_value.name is None
     assert isinstance(df.index_value.value, IndexValue.RangeIndex)
+    assert df.op.get_columns() == ["col1", "col2"]
     pd.testing.assert_series_equal(
         df.dtypes,
         pd.Series([np.dtype("O"), np.dtype("int64")], index=["col1", "col2"]),
@@ -257,6 +261,7 @@ def test_from_odps_table():
     assert df.index_value.name == "col1"
     assert isinstance(df.index_value.value, IndexValue.Index)
     assert df.index.dtype == np.dtype("O")
+    assert df.op.get_columns() == ["col2", "col3"]
     pd.testing.assert_series_equal(
         df.dtypes,
         pd.Series([np.dtype("int64"), np.dtype("float64")], index=["col2", "col3"]),
@@ -267,6 +272,7 @@ def test_from_odps_table():
 
     df = read_odps_table(test_parted_table, append_partitions=True)
     assert df.op.append_partitions is True
+    assert df.op.get_columns() == ["col1", "col2", "col3", "pt"]
     pd.testing.assert_series_equal(
         df.dtypes,
         pd.Series(
@@ -280,6 +286,7 @@ def test_from_odps_table():
     )
     assert df.op.append_partitions is True
     assert df.op.partitions == ["pt=20240103"]
+    assert df.op.get_columns() == ["col1", "col2", "pt"]
     pd.testing.assert_series_equal(
         df.dtypes,
         pd.Series(
@@ -377,3 +384,18 @@ def test_date_range():
     assert dr.index_value.is_unique == expected.is_unique
     assert dr.index_value.is_monotonic_increasing == expected.is_monotonic_increasing
     assert dr.name == expected.name
+
+
+def test_resolve_task_sector():
+    input_path = os.path.join(os.path.dirname(__file__), "test-data", "task-input.txt")
+    with open(input_path, "r") as f:
+        sector = f.read()
+    actual_sector = _resolve_task_sector("job0", sector)
+
+    assert actual_sector.job_name == "job0"
+    assert actual_sector.task_name == "M1"
+    assert actual_sector.output_target == "Screen"
+    assert len(actual_sector.schema) == 78
+    assert actual_sector.schema[0] == ColumnSchema("unnamed: 0", "bigint", "")
+    assert actual_sector.schema[1] == ColumnSchema("id", "bigint", "id_alias")
+    assert actual_sector.schema[2] == ColumnSchema("listing_url", "string", "")

{maxframe-0.1.0b2 → maxframe-0.1.0b3}/maxframe/dataframe/groupby/__init__.py

@@ -14,6 +14,7 @@
 
 # noinspection PyUnresolvedReferences
 from ..core import DataFrameGroupBy, GroupBy, SeriesGroupBy
+from .core import NamedAgg
 
 
 def _install():

{maxframe-0.1.0b2 → maxframe-0.1.0b3}/maxframe/dataframe/groupby/core.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from collections import namedtuple
+
 import pandas as pd
 
 from ... import opcodes
@@ -30,6 +32,9 @@ _GROUP_KEYS_NO_DEFAULT = pd_release_version >= (1, 5, 0)
 _default_group_keys = no_default if _GROUP_KEYS_NO_DEFAULT else True
 
 
+NamedAgg = namedtuple("NamedAgg", ["column", "aggfunc"])
+
+
 class DataFrameGroupByOperator(MapReduceOperator, DataFrameOperatorMixin):
     _op_type_ = opcodes.GROUPBY
 

{maxframe-0.1.0b2 → maxframe-0.1.0b3}/maxframe/lib/aio/isolation.py

@@ -14,11 +14,14 @@
 
 import asyncio
 import atexit
+import itertools
 import threading
 from typing import Dict, Optional
 
 
 class Isolation:
+    _counter = itertools.count().__next__
+
     loop: asyncio.AbstractEventLoop
     _stopped: Optional[asyncio.Event]
     _thread: Optional[threading.Thread]
@@ -38,7 +41,9 @@ class Isolation:
 
     def start(self):
         if self._threaded:
-            self._thread = thread = threading.Thread(target=self._run)
+            self._thread = thread = threading.Thread(
+                name=f"IsolationThread-{self._counter()}", target=self._run
+            )
             thread.daemon = True
             thread.start()
             self._thread_ident = thread.ident

{maxframe-0.1.0b2 → maxframe-0.1.0b3}/maxframe/protocol.py

@@ -46,6 +46,8 @@ BodyType = TypeVar("BodyType", bound="Serializable")
 
 
 class JsonSerializable(Serializable):
+    _ignore_non_existing_keys = True
+
     @classmethod
     def from_json(cls, serialized: dict) -> "JsonSerializable":
         raise NotImplementedError
@@ -245,6 +247,8 @@ class DagInfo(JsonSerializable):
         default_factory=dict,
     )
     error_info: Optional[ErrorInfo] = ReferenceField("error_info", default=None)
+    start_timestamp: Optional[float] = Float64Field("start_timestamp", default=None)
+    end_timestamp: Optional[float] = Float64Field("end_timestamp", default=None)
 
     @classmethod
     def from_json(cls, serialized: dict) -> "DagInfo":
@@ -265,7 +269,10 @@ class DagInfo(JsonSerializable):
             "dag_id": self.dag_id,
             "status": self.status.value,
             "progress": self.progress,
+            "start_timestamp": self.start_timestamp,
+            "end_timestamp": self.end_timestamp,
         }
+        ret = {k: v for k, v in ret.items() if v is not None}
         if self.tileable_to_result_infos:
             ret["tileable_to_result_infos"] = {
                 k: v.to_json() for k, v in self.tileable_to_result_infos.items()

{maxframe-0.1.0b2 → maxframe-0.1.0b3}/maxframe/serialization/serializables/core.py

@@ -112,6 +112,7 @@ class Serializable(metaclass=SerializableMeta):
     __slots__ = ("__weakref__",)
 
     _cache_primitive_serial = False
+    _ignore_non_existing_keys = False
 
     _FIELDS: Dict[str, Field]
     _FIELD_ORDER: List[str]
@@ -128,7 +129,11 @@ class Serializable(metaclass=SerializableMeta):
         else:
             values = kwargs
         for k, v in values.items():
-            fields[k].set(self, v)
+            try:
+                fields[k].set(self, v)
+            except KeyError:
+                if not self._ignore_non_existing_keys:
+                    raise
 
     def __on_deserialize__(self):
         pass

{maxframe-0.1.0b2 → maxframe-0.1.0b3}/maxframe/serialization/serializables/field.py

@@ -507,12 +507,14 @@ class ReferenceField(Field):
         tag: str,
         reference_type: Union[str, Type] = None,
         default: Any = no_default,
+        default_factory: Optional[Callable] = None,
         on_serialize: Callable[[Any], Any] = None,
         on_deserialize: Callable[[Any], Any] = None,
     ):
         super().__init__(
             tag,
             default=default,
+            default_factory=default_factory,
             on_serialize=on_serialize,
             on_deserialize=on_deserialize,
         )

{maxframe-0.1.0b2 → maxframe-0.1.0b3}/maxframe/tensor/core.py

@@ -43,7 +43,7 @@ from ..serialization.serializables import (
     StringField,
     TupleField,
 )
-from ..utils import on_deserialize_shape, on_serialize_shape
+from ..utils import on_deserialize_shape, on_serialize_shape, skip_na_call
 from .utils import fetch_corner_data, get_chunk_slices
 
 logger = logging.getLogger(__name__)
@@ -181,8 +181,8 @@ class TensorData(HasShapeTileableData, _ExecuteAndFetchMixin):
     _chunks = ListField(
         "chunks",
         FieldTypes.reference(TensorChunkData),
-        on_serialize=lambda x: [it.data for it in x] if x is not None else x,
-        on_deserialize=lambda x: [TensorChunk(it) for it in x] if x is not None else x,
+        on_serialize=skip_na_call(lambda x: [it.data for it in x]),
+        on_deserialize=skip_na_call(lambda x: [TensorChunk(it) for it in x]),
     )
 
     def __init__(

maxframe-0.1.0b3/maxframe/tests/test_codegen.py

@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+from typing import List, Tuple
+
+# 使用pytest生成单元测试
+import pytest
+
+from maxframe.codegen import UserCodeMixin
+from maxframe.lib import wrapped_pickle
+from maxframe.serialization.core import PickleContainer
+
+
+@pytest.mark.parametrize(
+    "input_obj, expected_output",
+    [
+        (None, "None"),
+        (10, "10"),
+        (3.14, "3.14"),
+        (True, "True"),
+        (False, "False"),
+        (b"hello", "base64.b64decode(b'aGVsbG8=')"),
+        ("hello", "'hello'"),
+        ([1, 2, 3], "[1, 2, 3]"),
+        ({"a": 1, "b": 2}, "{'a': 1, 'b': 2}"),
+        ((1, 2, 3), "(1, 2, 3)"),
+        ((1,), "(1,)"),
+        ((), "()"),
+        ({1, 2, 3}, "{1, 2, 3}"),
+        (set(), "set()"),
+    ],
+)
+def test_obj_to_python_expr(input_obj, expected_output):
+    assert UserCodeMixin.obj_to_python_expr(input_obj) == expected_output
+
+
+def test_obj_to_python_expr_custom_object():
+    class CustomClass:
+        def __init__(self, a: int, b: List[int], c: Tuple[int, int]):
+            self.a = a
+            self.b = b
+            self.c = c
+
+    custom_obj = CustomClass(1, [2, 3], (4, 5))
+    pickle_data = wrapped_pickle.dumps(custom_obj)
+    pickle_str = base64.b64encode(pickle_data)
+    custom_obj_pickle_container = PickleContainer([pickle_data])
+
+    # with class obj will not support currently
+    with pytest.raises(ValueError):
+        UserCodeMixin.obj_to_python_expr(custom_obj)
+
+    assert (
+        UserCodeMixin.obj_to_python_expr(custom_obj_pickle_container)
+        == f"cloudpickle.loads(base64.b64decode({pickle_str}), buffers=[])"
+    )

{maxframe-0.1.0b2 → maxframe-0.1.0b3}/maxframe/tests/test_protocol.py

@@ -11,6 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+import json
 import time
 
 import pytest
@@ -29,28 +31,32 @@ from ..serialization import RemoteException
 from ..utils import deserialize_serializable, serialize_serializable
 
 
+def _json_round_trip(json_data: dict) -> dict:
+    return json.loads(json.dumps(json_data))
+
+
 def test_result_info_json_serialize():
-    ri = ResultInfo.from_json(ResultInfo().to_json())
+    ri = ResultInfo.from_json(_json_round_trip(ResultInfo().to_json()))
     assert type(ri) is ResultInfo
 
     ri = ODPSTableResultInfo(
         full_table_name="table_name", partition_specs=["pt=partition"]
     )
-    deserial_ri = ResultInfo.from_json(ri.to_json())
+    deserial_ri = ResultInfo.from_json(_json_round_trip(ri.to_json()))
     assert type(ri) is ODPSTableResultInfo
     assert ri.result_type == deserial_ri.result_type
     assert ri.full_table_name == deserial_ri.full_table_name
     assert ri.partition_specs == deserial_ri.partition_specs
 
     ri = ODPSTableResultInfo(full_table_name="table_name")
-    deserial_ri = ResultInfo.from_json(ri.to_json())
+    deserial_ri = ResultInfo.from_json(_json_round_trip(ri.to_json()))
     assert type(ri) is ODPSTableResultInfo
     assert ri.result_type == deserial_ri.result_type
     assert ri.full_table_name == deserial_ri.full_table_name
     assert ri.partition_specs == deserial_ri.partition_specs
 
     ri = ODPSVolumeResultInfo(volume_name="vol_name", volume_path="vol_path")
-    deserial_ri = ResultInfo.from_json(ri.to_json())
+    deserial_ri = ResultInfo.from_json(_json_round_trip(ri.to_json()))
     assert type(ri) is ODPSVolumeResultInfo
     assert ri.result_type == deserial_ri.result_type
     assert ri.volume_name == deserial_ri.volume_name
@@ -63,7 +69,7 @@ def test_error_info_json_serialize():
     except ValueError as ex:
         err_info = ErrorInfo.from_exception(ex)
 
-    deserial_err_info = ErrorInfo.from_json(err_info.to_json())
+    deserial_err_info = ErrorInfo.from_json(_json_round_trip(err_info.to_json()))
     assert deserial_err_info.error_messages == err_info.error_messages
     assert isinstance(deserial_err_info.raw_error_data, ValueError)
 
@@ -73,7 +79,7 @@ def test_error_info_json_serialize():
     with pytest.raises(RemoteException):
         mf_err_info.reraise()
 
-    deserial_err_info = ErrorInfo.from_json(mf_err_info.to_json())
+    deserial_err_info = ErrorInfo.from_json(_json_round_trip(mf_err_info.to_json()))
     assert isinstance(deserial_err_info.raw_error_data, ValueError)
     with pytest.raises(ValueError):
         deserial_err_info.reraise()
@@ -94,7 +100,9 @@ def test_dag_info_json_serialize():
         },
         error_info=err_info,
     )
-    deserial_info = DagInfo.from_json(info.to_json())
+    json_info = info.to_json()
+    json_info["non_existing_field"] = "non_existing"
+    deserial_info = DagInfo.from_json(_json_round_trip(json_info))
     assert deserial_info.session_id == info.session_id
     assert deserial_info.dag_id == info.dag_id
     assert deserial_info.status == info.status
@@ -121,7 +129,7 @@ def test_session_info_json_serialize():
         idle_timestamp=None,
         dag_infos={"test_dag_id": dag_info},
     )
-    deserial_info = SessionInfo.from_json(info.to_json())
+    deserial_info = SessionInfo.from_json(_json_round_trip(info.to_json()))
     assert deserial_info.session_id == info.session_id
     assert deserial_info.settings == info.settings
     assert deserial_info.start_timestamp == info.start_timestamp

{maxframe-0.1.0b2 → maxframe-0.1.0b3}/maxframe/tests/utils.py

@@ -104,6 +104,7 @@ def run_app_in_thread(app_func):
         q = queue.Queue()
         exit_event = asyncio.Event(loop=app_loop)
         app_thread = Thread(
+            name="TestAppThread",
             target=app_thread_func,
             args=(app_loop, q, exit_event, args, kwargs),
         )