maxframe 0.1.0b5__cp39-cp39-macosx_10_9_universal2.whl → 1.0.0rc1__cp39-cp39-macosx_10_9_universal2.whl

maxframe/codegen.py

@@ -86,6 +86,8 @@ class AbstractUDF(Serializable):
 
 
 class UserCodeMixin:
+    __slots__ = ()
+
     @classmethod
     def obj_to_python_expr(cls, obj: Any = None) -> str:
         """
@@ -203,8 +205,12 @@ class BigDagCodeContext(metaclass=abc.ABCMeta):
         return self._session_id
 
     def register_udf(self, udf: AbstractUDF):
+        from maxframe_framedriver.services.session import SessionManager
+
         udf.session_id = self._session_id
         self._udfs[udf.name] = udf
+        if self._session_id and SessionManager.initialized():
+            SessionManager.instance().register_udf(self._session_id, udf)
 
     def get_udfs(self) -> List[AbstractUDF]:
         return list(self._udfs.values())
@@ -344,6 +350,8 @@ def register_engine_codegen(type_: Type["BigDagCodeGenerator"]):
 BUILTIN_ENGINE_SPE = "SPE"
 BUILTIN_ENGINE_MCSQL = "MCSQL"
 
+FAST_RANGE_INDEX_ENABLED = "codegen.fast_range_index_enabled"
+
 
 class BigDagCodeGenerator(metaclass=abc.ABCMeta):
     _context: BigDagCodeContext
@@ -516,12 +524,12 @@ class BigDagCodeGenerator(metaclass=abc.ABCMeta):
 
     def register_udfs(self, odps_ctx: "ODPSSessionContext"):
         for udf in self._context.get_udfs():
-            logger.info("[Session %s] Registering UDF %s", self._session_id, udf.name)
+            logger.info("[Session=%s] Registering UDF %s", self._session_id, udf.name)
             udf.register(odps_ctx, True)
 
     def unregister_udfs(self, odps_ctx: "ODPSSessionContext"):
         for udf in self._context.get_udfs():
-            logger.info("[Session %s] Unregistering UDF %s", self._session_id, udf.name)
+            logger.info("[Session=%s] Unregistering UDF %s", self._session_id, udf.name)
             udf.unregister(odps_ctx)
 
     def get_udfs(self) -> List[AbstractUDF]:

maxframe/config/config.py

@@ -40,6 +40,7 @@ _DEFAULT_SPE_OPERATION_TIMEOUT_SECONDS = 120
 _DEFAULT_UPLOAD_BATCH_SIZE = 4096
 _DEFAULT_TEMP_LIFECYCLE = 1
 _DEFAULT_TASK_START_TIMEOUT = 60
+_DEFAULT_TASK_RESTART_TIMEOUT = 300
 _DEFAULT_LOGVIEW_HOURS = 24 * 60
 
 
@@ -309,6 +310,9 @@ default_options.register_option(
 default_options.register_option(
     "client.task_start_timeout", _DEFAULT_TASK_START_TIMEOUT, validator=is_integer
 )
+default_options.register_option(
+    "client.task_restart_timeout", _DEFAULT_TASK_RESTART_TIMEOUT, validator=is_integer
+)
 default_options.register_option("sql.enable_mcqa", True, validator=is_bool, remote=True)
 default_options.register_option(
     "sql.generate_comments", True, validator=is_bool, remote=True

maxframe/core/__init__.py

@@ -19,7 +19,6 @@ from .entity import (
     CHUNK_TYPE,
     ENTITY_TYPE,
     FUSE_CHUNK_TYPE,
-    OBJECT_CHUNK_TYPE,
     OBJECT_TYPE,
     TILEABLE_TYPE,
     Chunk,
@@ -33,8 +32,6 @@ from .entity import (
     HasShapeTileableData,
     NotSupportTile,
     Object,
-    ObjectChunk,
-    ObjectChunkData,
     ObjectData,
     OutputType,
     Tileable,

maxframe/core/entity/__init__.py

@@ -16,14 +16,7 @@ from .chunks import CHUNK_TYPE, Chunk, ChunkData
 from .core import ENTITY_TYPE, Entity, EntityData
 from .executable import ExecutableTuple, _ExecuteAndFetchMixin
 from .fuse import FUSE_CHUNK_TYPE, FuseChunk, FuseChunkData
-from .objects import (
-    OBJECT_CHUNK_TYPE,
-    OBJECT_TYPE,
-    Object,
-    ObjectChunk,
-    ObjectChunkData,
-    ObjectData,
-)
+from .objects import OBJECT_TYPE, Object, ObjectData
 from .output_types import (
     OutputType,
     get_fetch_class,

maxframe/core/entity/objects.py

@@ -14,58 +14,17 @@
 
 from typing import Any, Dict
 
-from ...serialization.serializables import FieldTypes, ListField
-from ...utils import skip_na_call
-from .chunks import Chunk, ChunkData
 from .core import Entity
 from .executable import _ToObjectMixin
 from .tileables import TileableData
 
 
-class ObjectChunkData(ChunkData):
-    # chunk whose data could be any serializable
-    __slots__ = ()
-    type_name = "Object"
-
-    def __init__(self, op=None, index=None, **kw):
-        super().__init__(_op=op, _index=index, **kw)
-
-    @property
-    def params(self) -> Dict[str, Any]:
-        # params return the properties which useful to rebuild a new chunk
-        return {
-            "index": self.index,
-        }
-
-    @params.setter
-    def params(self, new_params: Dict[str, Any]):
-        params = new_params.copy()
-        params.pop("index", None)  # index not needed to update
-        if params:  # pragma: no cover
-            raise TypeError(f"Unknown params: {list(params)}")
-
-    @classmethod
-    def get_params_from_data(cls, data: Any) -> Dict[str, Any]:
-        return dict()
-
-
-class ObjectChunk(Chunk):
-    __slots__ = ()
-    _allow_data_type_ = (ObjectChunkData,)
-    type_name = "Object"
-
-
 class ObjectData(TileableData, _ToObjectMixin):
     __slots__ = ()
     type_name = "Object"
-
-    # optional fields
-    _chunks = ListField(
-        "chunks",
-        FieldTypes.reference(ObjectChunkData),
-        on_serialize=skip_na_call(lambda x: [it.data for it in x]),
-        on_deserialize=skip_na_call(lambda x: [ObjectChunk(it) for it in x]),
-    )
+    # workaround for removed field since v0.1.0b5
+    # todo remove this when all versions below v0.1.0b5 is eliminated
+    _legacy_deprecated_non_primitives = ["_chunks"]
 
     def __init__(self, op=None, nsplits=None, **kw):
         super().__init__(_op=op, _nsplits=nsplits, **kw)
@@ -97,4 +56,3 @@ class Object(Entity, _ToObjectMixin):
 
 
 OBJECT_TYPE = (Object, ObjectData)
-OBJECT_CHUNK_TYPE = (ObjectChunk, ObjectChunkData)

maxframe/core/graph/core.pyx

@@ -354,10 +354,10 @@ cdef class DirectedGraph:
                         sio.write(f'"Chunk:{self._gen_chunk_key(input_chunk, trunc_key)}" {chunk_style}\n')
                         visited.add(input_chunk.key)
                     if op.key not in visited:
-                        sio.write(f'"{op_name}:{op.key[:trunc_key]}" {operator_style}\n')
+                        sio.write(f'"{op_name}:{op.key[:trunc_key]}_{id(op)}" {operator_style}\n')
                         visited.add(op.key)
                     sio.write(f'"Chunk:{self._gen_chunk_key(input_chunk, trunc_key)}" -> '
-                              f'"{op_name}:{op.key[:trunc_key]}"\n')
+                              f'"{op_name}:{op.key[:trunc_key]}_{id(op)}"\n')
 
                 for output_chunk in (op.outputs or []):
                     if output_chunk.key not in visited:
@@ -367,9 +367,9 @@ cdef class DirectedGraph:
                         sio.write(f'"Chunk:{self._gen_chunk_key(output_chunk, trunc_key)}" {tmp_chunk_style}\n')
                         visited.add(output_chunk.key)
                     if op.key not in visited:
-                        sio.write(f'"{op_name}:{op.key[:trunc_key]}" {operator_style}\n')
+                        sio.write(f'"{op_name}:{op.key[:trunc_key]}_{id(op)}" {operator_style}\n')
                         visited.add(op.key)
-                    sio.write(f'"{op_name}:{op.key[:trunc_key]}" -> '
+                    sio.write(f'"{op_name}:{op.key[:trunc_key]}_{id(op)}" -> '
                               f'"Chunk:{self._gen_chunk_key(output_chunk, trunc_key)}"')
                     if show_columns:
                         sio.write(f' [ label={get_col_names(output_chunk)} ]')

maxframe/dataframe/datastore/tests/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

maxframe/dataframe/datastore/tests/test_to_odps.py

@@ -0,0 +1,48 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+from ... import DataFrame
+from ..to_odps import to_odps_table
+
+
+@pytest.fixture
+def df():
+    return DataFrame({"A": [1, 2], "B": [3, 4]})
+
+
+@pytest.mark.parametrize(
+    "kwargs",
+    [
+        {"partition_col": ["A", "C"]},
+        {"partition_col": "C"},
+        {"partition": "a=1,C=2"},
+    ],
+)
+def test_to_odps_table_validation(df, kwargs):
+    with pytest.raises(ValueError):
+        to_odps_table(df, "test_table", **kwargs)
+
+
+@pytest.mark.parametrize(
+    "kwargs",
+    [
+        {"partition_col": ["a", "B"]},
+        {"partition_col": "a"},
+        {"partition": "C=1,d=2"},
+    ],
+)
+def test_to_odps_table_vaild(df, kwargs):
+    to_odps_table(df, "test_table", **kwargs)

maxframe/dataframe/datastore/to_odps.py

@@ -18,10 +18,12 @@ import logging
 from typing import List, Optional, Union
 
 from odps.models import Table as ODPSTable
+from odps.types import PartitionSpec
 
 from ... import opcodes
 from ...config import options
 from ...core import OutputType
+from ...odpsio import build_dataframe_table_meta
 from ...serialization.serializables import (
     BoolField,
     FieldTypes,
@@ -147,6 +149,25 @@ def to_odps_table(
             f"index_label needs {len(df.index.nlevels)} labels "
             f"but it only have {len(index_label)}"
         )
+    table_cols = set(build_dataframe_table_meta(df).table_column_names)
+    if partition:
+        partition_intersect = (
+            set(x.lower() for x in PartitionSpec(partition).keys()) & table_cols
+        )
+        if partition_intersect:
+            raise ValueError(
+                f"Data column(s) {partition_intersect} in the dataframe"
+                " cannot be used in parameter 'partition'."
+                " Use 'partition_col' instead."
+            )
+
+    if partition_col:
+        partition_diff = set(x.lower() for x in partition_col) - table_cols
+        if partition_diff:
+            raise ValueError(
+                f"Partition column(s) {partition_diff}"
+                " is not the data column(s) of the input dataframe."
+            )
 
     op = DataFrameToODPSTable(
         dtypes=df.dtypes,

maxframe/dataframe/indexing/align.py

@@ -138,7 +138,7 @@ class DataFrameAlign(DataFrameOperator, DataFrameOperatorMixin):
                 series_index = rhs.index_value.to_pandas()
                 dtypes = lhs.dtypes.reindex(
                     lhs.dtypes.index.join(series_index, how=self.join)
-                ).fillna(np.dtype(np.float_))
+                ).fillna(np.dtype(float))
                 l_shape[1] = r_size = len(dtypes)
                 col_val = r_idx_val = parse_index(dtypes.index, store_data=True)
 

maxframe/dataframe/misc/apply.py

@@ -170,6 +170,8 @@ class ApplyOperator(
         elif self.output_types[0] == OutputType.dataframe:
             shape = [np.nan, np.nan]
             shape[1 - self.axis] = df.shape[1 - self.axis]
+            if self.axis == 1:
+                shape[1] = len(dtypes)
             shape = tuple(shape)
         else:
             shape = (df.shape[1 - self.axis],)

maxframe/dataframe/misc/memory_usage.py

@@ -58,7 +58,7 @@ class DataFrameMemoryUsage(DataFrameOperator, DataFrameOperatorMixin):
         """
         if df_or_series.ndim == 1:
             # the input data is a series, a Scalar will be returned
-            return self.new_scalar([df_or_series], dtype=np.dtype(np.int_))
+            return self.new_scalar([df_or_series], dtype=np.dtype(int))
         else:
             # the input data is a DataFrame, a Scalar will be returned
             # calculate shape of returning series given ``op.index``
@@ -71,7 +71,7 @@ class DataFrameMemoryUsage(DataFrameOperator, DataFrameOperatorMixin):
                 [df_or_series],
                 index_value=self._adapt_index(df_or_series.columns_value),
                 shape=new_shape,
-                dtype=np.dtype(np.int_),
+                dtype=np.dtype(int),
             )
 
 

maxframe/dataframe/misc/tests/test_misc.py

@@ -18,6 +18,7 @@ import pytest
 
 from .... import opcodes
 from ....core import OutputType
+from ....dataframe import DataFrame
 from ....tensor.core import TENSOR_TYPE
 from ... import eval as maxframe_eval
 from ... import get_dummies, to_numeric
@@ -430,6 +431,28 @@ def test_case_when():
     assert isinstance(col.inputs[2].op, DataFrameGreater)
 
 
+def test_apply():
+    df = DataFrame({"a": [1, 2, 3], "b": [1, 2, 3], "c": [1, 2, 3]})
+
+    keys = [1, 2]
+
+    def f(x, keys):
+        if x["a"] in keys:
+            return [1, 0]
+        else:
+            return [0, 1]
+
+    apply_df = df[["a"]].apply(
+        f,
+        output_type="dataframe",
+        dtypes=pd.Series(["int64", "int64"]),
+        axis=1,
+        result_type="expand",
+        keys=keys,
+    )
+    assert apply_df.shape == (3, 2)
+
+
 def test_pivot_table():
     from ...groupby.aggregation import DataFrameGroupByAgg
     from ...misc.pivot_table import DataFramePivotTable

maxframe/dataframe/statistics/corr.py

@@ -43,7 +43,7 @@ class DataFrameCorr(DataFrameOperator, DataFrameOperatorMixin):
     def __call__(self, df_or_series):
         if isinstance(df_or_series, SERIES_TYPE):
             inputs = filter_inputs([df_or_series, self.other])
-            return self.new_scalar(inputs, dtype=np.dtype(np.float_))
+            return self.new_scalar(inputs, dtype=np.dtype(float))
         else:
 
             def _filter_numeric(obj):
@@ -60,7 +60,7 @@ class DataFrameCorr(DataFrameOperator, DataFrameOperatorMixin):
             inputs = filter_inputs([df_or_series, self.other])
             if self.axis is None:
                 dtypes = pd.Series(
-                    [np.dtype(np.float_)] * len(df_or_series.dtypes),
+                    [np.dtype(float)] * len(df_or_series.dtypes),
                     index=df_or_series.dtypes.index,
                 )
                 return self.new_dataframe(
@@ -85,7 +85,7 @@ class DataFrameCorr(DataFrameOperator, DataFrameOperatorMixin):
                 return self.new_series(
                     inputs,
                     shape=shape,
-                    dtype=np.dtype(np.float_),
+                    dtype=np.dtype(float),
                     index_value=new_index_value,
                 )
 

maxframe/errors.py

@@ -17,5 +17,18 @@ class MaxFrameError(Exception):
     pass
 
 
+class MaxFrameIntentionalError(MaxFrameError):
+    pass
+
+
 class MaxFrameUserError(MaxFrameError):
     pass
+
+
+class NoTaskServerResponseError(MaxFrameError):
+    pass
+
+
+class SessionAlreadyClosedError(MaxFrameError):
+    def __init__(self, session_id: str):
+        super().__init__(f"Session {session_id} is already closed")

maxframe/extension.py

@@ -48,6 +48,18 @@ class MaxFrameExtension(metaclass=abc.ABCMeta):
         """
         pass
 
+    @classmethod
+    async def reload_session(cls, session_id: str) -> None:
+        """
+        Reload the session state when the session is recovered from failover.
+
+        Parameters
+        ----------
+        session_id : str
+            The session id.
+        """
+        pass
+
     @classmethod
     def init_service_extension(cls) -> None:
         """

maxframe/lib/mmh3.pyi

@@ -0,0 +1,43 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Tuple
+
+def hash(key, seed=0, signed=True) -> int:
+    """
+    Return a 32 bit integer.
+    """
+
+def hash_from_buffer(key, seed=0, signed=True) -> int:
+    """
+    Return a 32 bit integer. Designed for large memory-views such as numpy arrays.
+    """
+
+def hash64(key, seed=0, x64arch=True, signed=True) -> Tuple[int, int]:
+    """
+    Return a tuple of two 64 bit integers for a string. Optimized for
+    the x64 bit architecture when x64arch=True, otherwise for x86.
+    """
+
+def hash128(key, seed=0, x64arch=True, signed=False) -> int:
+    """
+    Return a 128 bit long integer. Optimized for the x64 bit architecture
+    when x64arch=True, otherwise for x86.
+    """
+
+def hash_bytes(key, seed=0, x64arch=True) -> bytes:
+    """
+    Return a 128 bit hash value as bytes for a string. Optimized for the
+    x64 bit architecture when x64arch=True, otherwise for the x86.
+    """

maxframe/lib/wrapped_pickle.py

@@ -120,7 +120,8 @@ class _UnpickleSwitch:
             @functools.wraps(func)
             async def wrapped(*args, **kwargs):
                 with _UnpickleSwitch(forbidden=self._forbidden):
-                    return await func(*args, **kwargs)
+                    ret = await func(*args, **kwargs)
+                return ret
 
         else: