maxframe 0.1.0b4__cp310-cp310-macosx_10_9_universal2.whl → 1.0.0__cp310-cp310-macosx_10_9_universal2.whl

maxframe/protocol.py

@@ -15,7 +15,7 @@
 import base64
 import enum
 import uuid
-from typing import Any, Dict, Generic, List, Optional, Tuple, Type, TypeVar
+from typing import Any, Dict, Generic, List, Optional, Type, TypeVar
 
 import pandas as pd
 
@@ -32,12 +32,12 @@ from .serialization.serializables import (
     EnumField,
     FieldTypes,
     Float64Field,
+    Int32Field,
     ListField,
     ReferenceField,
     Serializable,
     SeriesField,
     StringField,
-    TupleField,
 )
 
 pickling_support.install()
@@ -71,6 +71,9 @@ class DagStatus(enum.Enum):
     CANCELLING = 4
     CANCELLED = 5
 
+    def is_terminated(self):
+        return self in (DagStatus.CANCELLED, DagStatus.SUCCEEDED, DagStatus.FAILED)
+
 
 class DimensionIndex(Serializable):
     is_slice: bool = BoolField("is_slice", default=None)
@@ -88,19 +91,6 @@ class DataSerializeType(enum.Enum):
     PICKLE = 0
 
 
-class VolumeDataMeta(Serializable):
-    output_type: OutputType = EnumField(
-        "output_type", OutputType, FieldTypes.int8, default=None
-    )
-    serial_type: DataSerializeType = EnumField(
-        "serial_type", DataSerializeType, FieldTypes.int8, default=None
-    )
-    shape: Tuple[int, ...] = TupleField("shape", FieldTypes.int64, default=None)
-    nsplits: Tuple[Tuple[int, ...], ...] = TupleField(
-        "nsplits", FieldTypes.tuple(FieldTypes.tuple(FieldTypes.int64)), default=None
-    )
-
-
 _result_type_to_info_cls: Dict[ResultType, Type["ResultInfo"]] = dict()
 
 
@@ -150,6 +140,9 @@ class ODPSTableResultInfo(ResultInfo):
     partition_specs: Optional[List[str]] = ListField(
         "partition_specs", FieldTypes.string, default=None
     )
+    table_meta: Optional["DataFrameTableMeta"] = ReferenceField(
+        "table_meta", default=None
+    )
 
     def __init__(self, result_type: ResultType = None, **kw):
         result_type = result_type or ResultType.ODPS_TABLE
@@ -160,8 +153,17 @@ class ODPSTableResultInfo(ResultInfo):
         ret["full_table_name"] = self.full_table_name
         if self.partition_specs:
             ret["partition_specs"] = self.partition_specs
+        if self.table_meta:
+            ret["table_meta"] = self.table_meta.to_json()
         return ret
 
+    @classmethod
+    def _json_to_kwargs(cls, serialized: dict) -> dict:
+        kw = super()._json_to_kwargs(serialized)
+        if "table_meta" in kw:
+            kw["table_meta"] = DataFrameTableMeta.from_json(kw["table_meta"])
+        return kw
+
 
 class ODPSVolumeResultInfo(ResultInfo):
     _result_type = ResultType.ODPS_VOLUME
@@ -190,9 +192,9 @@ class ErrorInfo(JsonSerializable):
         "error_tracebacks", FieldTypes.list
     )
     raw_error_source: ErrorSource = EnumField(
-        "raw_error_source", ErrorSource, FieldTypes.int8
+        "raw_error_source", ErrorSource, FieldTypes.int8, default=None
     )
-    raw_error_data: Optional[Exception] = AnyField("raw_error_data")
+    raw_error_data: Optional[Exception] = AnyField("raw_error_data", default=None)
 
     @classmethod
     def from_exception(cls, exc: Exception):
@@ -201,20 +203,29 @@ class ErrorInfo(JsonSerializable):
         return cls(messages, tracebacks, ErrorSource.PYTHON, exc)
 
     def reraise(self):
-        if self.raw_error_source == ErrorSource.PYTHON:
+        if (
+            self.raw_error_source == ErrorSource.PYTHON
+            and self.raw_error_data is not None
+        ):
             raise self.raw_error_data
         raise RemoteException(self.error_messages, self.error_tracebacks, [])
 
     @classmethod
     def from_json(cls, serialized: dict) -> "ErrorInfo":
         kw = serialized.copy()
-        kw["raw_error_source"] = ErrorSource(serialized["raw_error_source"])
+        if kw.get("raw_error_source") is not None:
+            kw["raw_error_source"] = ErrorSource(serialized["raw_error_source"])
+        else:
+            kw["raw_error_source"] = None
+
         if kw.get("raw_error_data"):
             bufs = [base64.b64decode(s) for s in kw["raw_error_data"]]
             try:
                 kw["raw_error_data"] = pickle.loads(bufs[0], buffers=bufs[1:])
             except:
-                kw["raw_error_data"] = None
+                # both error source and data shall be None to make sure
+                # RemoteException is raised.
+                kw["raw_error_source"] = kw["raw_error_data"] = None
         return cls(**kw)
 
     def to_json(self) -> dict:
@@ -227,7 +238,12 @@ class ErrorInfo(JsonSerializable):
         if isinstance(self.raw_error_data, (PickleContainer, RemoteException)):
             err_data_bufs = self.raw_error_data.get_buffers()
         elif isinstance(self.raw_error_data, BaseException):
-            err_data_bufs = pickle_buffers(self.raw_error_data)
+            try:
+                err_data_bufs = pickle_buffers(self.raw_error_data)
+            except:
+                err_data_bufs = None
+                ret["raw_error_source"] = None
+
         if err_data_bufs:
             ret["raw_error_data"] = [
                 base64.b64encode(s).decode() for s in err_data_bufs
@@ -249,9 +265,17 @@ class DagInfo(JsonSerializable):
     error_info: Optional[ErrorInfo] = ReferenceField("error_info", default=None)
     start_timestamp: Optional[float] = Float64Field("start_timestamp", default=None)
     end_timestamp: Optional[float] = Float64Field("end_timestamp", default=None)
+    subdag_infos: Dict[str, "SubDagInfo"] = DictField(
+        "subdag_infos",
+        key_type=FieldTypes.string,
+        value_type=FieldTypes.reference,
+        default_factory=dict,
+    )
 
     @classmethod
-    def from_json(cls, serialized: dict) -> "DagInfo":
+    def from_json(cls, serialized: dict) -> Optional["DagInfo"]:
+        if serialized is None:
+            return None
         kw = serialized.copy()
         kw["status"] = DagStatus(kw["status"])
         if kw.get("tileable_to_result_infos"):
@@ -261,6 +285,10 @@ class DagInfo(JsonSerializable):
             }
         if kw.get("error_info"):
             kw["error_info"] = ErrorInfo.from_json(kw["error_info"])
+        if kw.get("subdag_infos"):
+            kw["subdag_infos"] = {
+                k: SubDagInfo.from_json(v) for k, v in kw["subdag_infos"].items()
+            }
         return DagInfo(**kw)
 
     def to_json(self) -> dict:
@@ -279,6 +307,8 @@ class DagInfo(JsonSerializable):
             }
         if self.error_info:
             ret["error_info"] = self.error_info.to_json()
+        if self.subdag_infos:
+            ret["subdag_infos"] = {k: v.to_json() for k, v in self.subdag_infos.items()}
         return ret
 
 
@@ -302,7 +332,9 @@ class SessionInfo(JsonSerializable):
     error_info: Optional[ErrorInfo] = ReferenceField("error_info", default=None)
 
     @classmethod
-    def from_json(cls, serialized: dict) -> "SessionInfo":
+    def from_json(cls, serialized: dict) -> Optional["SessionInfo"]:
+        if serialized is None:
+            return None
         kw = serialized.copy()
         if kw.get("dag_infos"):
             kw["dag_infos"] = {
@@ -320,7 +352,10 @@ class SessionInfo(JsonSerializable):
             "idle_timestamp": self.idle_timestamp,
         }
         if self.dag_infos:
-            ret["dag_infos"] = {k: v.to_json() for k, v in self.dag_infos.items()}
+            ret["dag_infos"] = {
+                k: v.to_json() if v is not None else None
+                for k, v in self.dag_infos.items()
+            }
         if self.error_info:
             ret["error_info"] = self.error_info.to_json()
         return ret
@@ -340,9 +375,32 @@ class ExecuteDagRequest(Serializable):
         value_type=FieldTypes.reference,
         default=None,
     )
+    new_settings: Dict[str, Any] = DictField(
+        "new_settings",
+        key_type=FieldTypes.string,
+        default=None,
+    )
+
+
+class SubDagSubmitInstanceInfo(JsonSerializable):
+    submit_reason: str = StringField("submit_reason")
+    instance_id: str = StringField("instance_id")
+    subquery_id: Optional[int] = Int32Field("subquery_id", default=None)
+
+    @classmethod
+    def from_json(cls, serialized: dict) -> "SubDagSubmitInstanceInfo":
+        return SubDagSubmitInstanceInfo(**serialized)
+
+    def to_json(self) -> dict:
+        ret = {
+            "submit_reason": self.submit_reason,
+            "instance_id": self.instance_id,
+            "subquery_id": self.subquery_id,
+        }
+        return ret
 
 
-class SubDagInfo(Serializable):
+class SubDagInfo(JsonSerializable):
     subdag_id: str = StringField("subdag_id")
     status: DagStatus = EnumField("status", DagStatus, FieldTypes.int8, default=None)
     progress: float = Float64Field("progress", default=None)
@@ -355,9 +413,52 @@ class SubDagInfo(Serializable):
         FieldTypes.reference,
         default_factory=dict,
     )
+    start_timestamp: Optional[float] = Float64Field("start_timestamp", default=None)
+    end_timestamp: Optional[float] = Float64Field("end_timestamp", default=None)
+    submit_instances: List[SubDagSubmitInstanceInfo] = ListField(
+        "submit_instances",
+        FieldTypes.reference,
+        default_factory=list,
+    )
+
+    @classmethod
+    def from_json(cls, serialized: dict) -> "SubDagInfo":
+        kw = serialized.copy()
+        kw["status"] = DagStatus(kw["status"])
+        if kw.get("tileable_to_result_infos"):
+            kw["tileable_to_result_infos"] = {
+                k: ResultInfo.from_json(s)
+                for k, s in kw["tileable_to_result_infos"].items()
+            }
+        if kw.get("error_info"):
+            kw["error_info"] = ErrorInfo.from_json(kw["error_info"])
+        if kw.get("submit_instances"):
+            kw["submit_instances"] = [
+                SubDagSubmitInstanceInfo.from_json(s) for s in kw["submit_instances"]
+            ]
+        return SubDagInfo(**kw)
+
+    def to_json(self) -> dict:
+        ret = {
+            "subdag_id": self.subdag_id,
+            "status": self.status.value,
+            "progress": self.progress,
+            "start_timestamp": self.start_timestamp,
+            "end_timestamp": self.end_timestamp,
+        }
+        if self.error_info:
+            ret["error_info"] = self.error_info.to_json()
+        if self.tileable_to_result_infos:
+            ret["tileable_to_result_infos"] = {
+                k: v.to_json() for k, v in self.tileable_to_result_infos.items()
+            }
+        if self.submit_instances:
+            ret["submit_instances"] = [i.to_json() for i in self.submit_instances]
+        return ret
 
 
 class ExecuteSubDagRequest(Serializable):
+    subdag_id: str = StringField("subdag_id")
     dag: TileableGraph = ReferenceField(
         "dag",
         on_serialize=SerializableGraph.from_graph,
@@ -371,7 +472,7 @@ class DecrefRequest(Serializable):
     keys: List[str] = ListField("keys", FieldTypes.string, default=None)
 
 
-class DataFrameTableMeta(Serializable):
+class DataFrameTableMeta(JsonSerializable):
     __slots__ = "_pd_column_names", "_pd_index_level_names"
 
     table_name: Optional[str] = StringField("table_name", default=None)
@@ -402,7 +503,7 @@ class DataFrameTableMeta(Serializable):
             self._pd_index_level_names = self.pd_index_dtypes.index.tolist()
             return self._pd_index_level_names
 
-    def __eq__(self, other: "Serializable") -> bool:
+    def __eq__(self, other: "DataFrameTableMeta") -> bool:
         if not isinstance(other, type(self)):
             return False
         for k in self._FIELDS:
@@ -413,3 +514,29 @@ class DataFrameTableMeta(Serializable):
             if not is_same:
                 return False
         return True
+
+    def to_json(self) -> dict:
+        b64_pk = lambda x: base64.b64encode(pickle.dumps(x)).decode()
+        ret = {
+            "table_name": self.table_name,
+            "type": self.type.value,
+            "table_column_names": self.table_column_names,
+            "table_index_column_names": self.table_index_column_names,
+            "pd_column_dtypes": b64_pk(self.pd_column_dtypes),
+            "pd_column_level_names": b64_pk(self.pd_column_level_names),
+            "pd_index_dtypes": b64_pk(self.pd_index_dtypes),
+        }
+        return ret
+
+    @classmethod
+    def from_json(cls, serialized: dict) -> "DataFrameTableMeta":
+        b64_upk = lambda x: pickle.loads(base64.b64decode(x))
+        serialized.update(
+            {
+                "type": OutputType(serialized["type"]),
+                "pd_column_dtypes": b64_upk(serialized["pd_column_dtypes"]),
+                "pd_column_level_names": b64_upk(serialized["pd_column_level_names"]),
+                "pd_index_dtypes": b64_upk(serialized["pd_index_dtypes"]),
+            }
+        )
+        return DataFrameTableMeta(**serialized)

maxframe/remote/core.py

@@ -15,7 +15,7 @@
 from functools import partial
 
 from .. import opcodes
-from ..core import ENTITY_TYPE, ChunkData
+from ..core import ENTITY_TYPE
 from ..core.operator import ObjectOperator, ObjectOperatorMixin
 from ..dataframe.core import DATAFRAME_TYPE, INDEX_TYPE, SERIES_TYPE
 from ..serialization.serializables import (
@@ -26,7 +26,7 @@ from ..serialization.serializables import (
     ListField,
 )
 from ..tensor.core import TENSOR_TYPE
-from ..utils import build_fetch_tileable, find_objects, replace_objects
+from ..utils import find_objects, replace_objects
 
 
 class RemoteFunction(ObjectOperatorMixin, ObjectOperator):
@@ -63,12 +63,8 @@ class RemoteFunction(ObjectOperatorMixin, ObjectOperator):
         if raw_inputs is not None:
             for raw_inp in raw_inputs:
                 if self._no_prepare(raw_inp):
-                    if not isinstance(self._inputs[0], ChunkData):
-                        # not in tile, set_inputs from tileable
-                        mapping[raw_inp] = next(function_inputs)
-                    else:
-                        # in tile, set_inputs from chunk
-                        mapping[raw_inp] = build_fetch_tileable(raw_inp)
+                    # not in tile, set_inputs from tileable
+                    mapping[raw_inp] = next(function_inputs)
                 else:
                     mapping[raw_inp] = next(function_inputs)
         self.function_args = replace_objects(self.function_args, mapping)

maxframe/serialization/__init__.py

@@ -17,6 +17,7 @@ from .core import (
     PickleContainer,
     Serializer,
     deserialize,
+    load_type,
     pickle_buffers,
     serialize,
     serialize_with_spawn,

maxframe/serialization/core.pxd

@@ -18,6 +18,9 @@ from libc.stdint cimport int32_t, uint64_t
 cdef class Serializer:
     cdef int _serializer_id
 
+    cpdef bint is_public_data_exist(self, dict context, object key)
+    cpdef put_public_data(self, dict context, object key, object value)
+    cpdef get_public_data(self, dict context, object key)
     cpdef serial(self, object obj, dict context)
     cpdef deserial(self, list serialized, dict context, list subs)
     cpdef on_deserial_error(

maxframe/serialization/core.pyi

@@ -0,0 +1,64 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from concurrent.futures import Executor
+from typing import Any, Callable, Dict, List, TypeVar
+
+def buffered(func: Callable) -> Callable: ...
+def fast_id(obj: Any) -> int: ...
+
+LoadType = TypeVar("LoadType")
+
+def load_type(class_name: str, parent_class: LoadType) -> LoadType: ...
+
+class PickleContainer:
+    def __init__(self, buffers: List[bytes]): ...
+    def get(self) -> Any: ...
+    def get_buffers(self) -> List[bytes]: ...
+
+class Serializer:
+    serializer_id: int
+    def is_public_data_exist(self, context: Dict, key: Any) -> bool: ...
+    def put_public_data(self, context: Dict, key: Any, value: Any) -> None: ...
+    def get_public_data(self, context: Dict, key: Any) -> Any: ...
+    def serial(self, obj: Any, context: Dict): ...
+    def deserial(self, serialized: List, context: Dict, subs: List[Any]): ...
+    def on_deserial_error(
+        self,
+        serialized: List,
+        context: Dict,
+        subs_serialized: List,
+        error_index: int,
+        exc: BaseException,
+    ): ...
+    @classmethod
+    def register(cls, obj_type): ...
+    @classmethod
+    def unregister(cls, obj_type): ...
+
+class Placeholder:
+    id: int
+    callbacks: List[Callable]
+    def __init__(self, id_: int): ...
+    def __hash__(self): ...
+    def __eq__(self, other): ...
+
+def serialize(obj: Any, context: Dict = None): ...
+async def serialize_with_spawn(
+    obj: Any,
+    context: Dict = None,
+    spawn_threshold: int = 100,
+    executor: Executor = None,
+): ...
+def deserialize(headers: List, buffers: List, context: Dict = None): ...

maxframe/serialization/core.pyx

@@ -37,7 +37,7 @@ from .._utils import NamedType
 from .._utils cimport TypeDispatcher
 
 from ..lib import wrapped_pickle as pickle
-from ..utils import arrow_type_from_str
+from ..utils import NoDefault, arrow_type_from_str, no_default
 
 try:
     from pandas import ArrowDtype
@@ -94,6 +94,7 @@ cdef:
     int COMPLEX_SERIALIZER = 12
     int SLICE_SERIALIZER = 13
     int REGEX_SERIALIZER = 14
+    int NO_DEFAULT_SERIALIZER = 15
     int PLACEHOLDER_SERIALIZER = 4096
 
 
@@ -130,11 +131,30 @@ cdef Serializer get_deserializer(int32_t deserializer_id):
 
 cdef class Serializer:
     serializer_id = None
+    _public_data_context_key = 0x7fffffff - 1
 
     def __cinit__(self):
         # make the value can be referenced with C code
         self._serializer_id = self.serializer_id
 
+    cpdef bint is_public_data_exist(self, dict context, object key):
+        cdef dict public_dict = context.get(self._public_data_context_key, None)
+        if public_dict is None:
+            return False
+        return key in public_dict
+
+    cpdef put_public_data(self, dict context, object key, object value):
+        cdef dict public_dict = context.get(self._public_data_context_key, None)
+        if public_dict is None:
+            public_dict = context[self._public_data_context_key] = {}
+        public_dict[key] = value
+
+    cpdef get_public_data(self, dict context, object key):
+        cdef dict public_dict = context.get(self._public_data_context_key, None)
+        if public_dict is None:
+            return None
+        return public_dict.get(key)
+
     cpdef serial(self, object obj, dict context):
         """
         Returns intermediate serialization result of certain object.
@@ -784,6 +804,16 @@ cdef class RegexSerializer(Serializer):
         return re.compile((<bytes>(subs[0])).decode(), serialized[0])
 
 
+cdef class NoDefaultSerializer(Serializer):
+    serializer_id = NO_DEFAULT_SERIALIZER
+
+    cpdef serial(self, object obj, dict context):
+        return [], [], True
+
+    cpdef deserial(self, list obj, dict context, list subs):
+        return no_default
+
+
 cdef class Placeholder:
     """
     Placeholder object to reduce duplicated serialization
@@ -838,6 +868,7 @@ DtypeSerializer.register(ExtensionDtype)
 ComplexSerializer.register(complex)
 SliceSerializer.register(slice)
 RegexSerializer.register(re.Pattern)
+NoDefaultSerializer.register(NoDefault)
 PlaceholderSerializer.register(Placeholder)
 
 
@@ -993,17 +1024,20 @@ def serialize(obj, dict context = None):
     cdef list subs
     cdef bint final
     cdef _IdContextHolder id_context_holder = _IdContextHolder()
+    cdef tuple result
 
     context = context if context is not None else dict()
     serialized, subs, final = _serial_single(obj, context, id_context_holder)
     if final or not subs:
         # marked as a leaf node, return directly
-        return [{}, serialized], subs
-
-    serial_stack.append(_SerialStackItem(serialized, subs))
-    return _serialize_with_stack(
-        serial_stack, None, context, id_context_holder, result_bufs_list
-    )
+        result = [{}, serialized], subs
+    else:
+        serial_stack.append(_SerialStackItem(serialized, subs))
+        result = _serialize_with_stack(
+            serial_stack, None, context, id_context_holder, result_bufs_list
+        )
+    result[0][0]["_PUB"] = context.get(Serializer._public_data_context_key)
+    return result
 
 
 async def serialize_with_spawn(
@@ -1036,31 +1070,38 @@ async def serialize_with_spawn(
     cdef list subs
     cdef bint final
     cdef _IdContextHolder id_context_holder = _IdContextHolder()
+    cdef tuple result
 
     context = context if context is not None else dict()
     serialized, subs, final = _serial_single(obj, context, id_context_holder)
     if final or not subs:
         # marked as a leaf node, return directly
-        return [{}, serialized], subs
-
-    serial_stack.append(_SerialStackItem(serialized, subs))
+        result = [{}, serialized], subs
+    else:
+        serial_stack.append(_SerialStackItem(serialized, subs))
 
-    try:
-        result = _serialize_with_stack(
-            serial_stack, None, context, id_context_holder, result_bufs_list, spawn_threshold
-        )
-    except _SerializeObjectOverflow as ex:
-        result = await asyncio.get_running_loop().run_in_executor(
-            executor,
-            _serialize_with_stack,
-            serial_stack,
-            ex.cur_serialized,
-            context,
-            id_context_holder,
-            result_bufs_list,
-            0,
-            ex.num_total_serialized,
-        )
+        try:
+            result = _serialize_with_stack(
+                serial_stack,
+                None,
+                context,
+                id_context_holder,
+                result_bufs_list,
+                spawn_threshold,
+            )
+        except _SerializeObjectOverflow as ex:
+            result = await asyncio.get_running_loop().run_in_executor(
+                executor,
+                _serialize_with_stack,
+                serial_stack,
+                ex.cur_serialized,
+                context,
+                id_context_holder,
+                result_bufs_list,
+                0,
+                ex.num_total_serialized,
+            )
+    result[0][0]["_PUB"] = context.get(Serializer._public_data_context_key)
     return result
 
 

maxframe/serialization/exception.py

@@ -35,7 +35,7 @@ class RemoteException(MaxFrameError):
     def from_exception(cls, exc: Exception):
         try:
             buffers = pickle_buffers(exc)
-        except (TypeError, pickle.PicklingError):
+        except:
             logger.exception("Cannot pickle exception %s", exc)
             buffers = []