maxframe 2.2.0__cp38-cp38-win_amd64.whl → 2.3.0rc1__cp38-cp38-win_amd64.whl

maxframe/codegen/core.py

@@ -23,7 +23,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union
 from odps.types import OdpsSchema
 from odps.utils import camel_to_underline
 
-from ..core import OperatorType, Tileable, TileableGraph
+from ..core import OperatorType, Tileable, TileableGraph, enter_mode
 from ..core.operator import Fetch, Operator
 from ..extension import iter_extensions
 from ..io.odpsio import build_dataframe_table_meta
@@ -465,6 +465,7 @@ class DAGCodeGenerator(metaclass=abc.ABCMeta):
     def _generate_delete_code(self, var_name: str) -> List[str]:
         return []
 
+    @enter_mode(build=True)
     def generate_code(self, dag: TileableGraph) -> List[str]:
         """
         Generate the code of the input dag.
@@ -494,7 +495,7 @@ class DAGCodeGenerator(metaclass=abc.ABCMeta):
             code_lines.extend(adapter.generate_pre_op_code(op, self._context))
             if self._generate_comments_enabled:
                 code_lines.extend(adapter.generate_comment(op, self._context))
-            code_lines.extend(adapter.generate_code(op, self._context))
+            code_lines.extend(adapter.generate_code(op, self._context) or [])
             code_lines.extend(adapter.generate_post_op_code(op, self._context))
             code_lines.append("")  # Append an empty line to separate operators
 

maxframe/codegen/spe/dataframe/merge.py

@@ -15,6 +15,7 @@
 from typing import List
 
 from ....dataframe.merge import (
+    DataFrameCombine,
     DataFrameCompare,
     DataFrameConcat,
     DataFrameMerge,
@@ -23,6 +24,9 @@ from ....dataframe.merge import (
 from ..core import SPECodeContext, SPEOperatorAdapter, register_op_adapter
 from ..utils import build_method_call_adapter
 
+DataFrameCombineAdapter = build_method_call_adapter(
+    DataFrameCombine, "combine", 1, "func", kw_keys=["fill_value"]
+)
 DataFrameCompareAdapter = build_method_call_adapter(
     DataFrameCompare,
     "compare",

maxframe/codegen/spe/dataframe/misc.py

@@ -32,6 +32,7 @@ from ....dataframe.misc.get_dummies import DataFrameGetDummies
 from ....dataframe.misc.isin import DataFrameIsin
 from ....dataframe.misc.map import DataFrameMap
 from ....dataframe.misc.memory_usage import DataFrameMemoryUsage
+from ....dataframe.misc.repeat import DataFrameRepeat
 from ....dataframe.misc.shift import DataFrameShift
 from ....dataframe.misc.to_numeric import DataFrameToNumeric
 from ....dataframe.misc.transform import DataFrameTransform
@@ -127,6 +128,7 @@ class DataFrameIsinAdapter(SPEOperatorAdapter):
 DataFrameMapAdapter = build_method_call_adapter(
     DataFrameMap, "map", "arg", na_action=None
 )
+DataFrameRepeatAdapter = build_method_call_adapter(DataFrameRepeat, "repeat", "repeats")
 DataFrameShiftAdapter = build_method_call_adapter(
     DataFrameShift, "shift", kw_keys=["periods", "freq", "axis", "fill_value"]
 )

maxframe/codegen/spe/dataframe/reduction.py

@@ -28,6 +28,7 @@ from ....dataframe.reduction import (
     DataFrameMean,
     DataFrameMedian,
     DataFrameMin,
+    DataFrameMode,
     DataFrameNunique,
     DataFrameProd,
     DataFrameSem,
@@ -163,3 +164,20 @@ class DataFrameUniqueAdapter(SPEOperatorAdapter):
         input_var_name = context.get_input_tileable_variable(op.inputs[0])
         res_var_name = context.get_output_tileable_variable(op.outputs[0])
         return [f"{res_var_name} = pd.unique({input_var_name})"]
+
+
+@register_op_adapter(DataFrameMode)
+class DataFrameModeAdapter(SPEOperatorAdapter):
+    def generate_code(self, op: DataFrameMode, context: SPECodeContext) -> List[str]:
+        input_var_name = context.get_input_tileable_variable(op.inputs[0])
+        args = []
+        if op.inputs[0].ndim == 2:
+            if op.axis is not None:
+                args.append(f"axis={op.axis!r}")
+            if op.numeric_only is not None:
+                args.append(f"numeric_only={op.numeric_only!r}")
+        if op.dropna is not None:
+            args.append(f"dropna={op.dropna!r}")
+        args_str = ", ".join(args)
+        res_var_name = context.get_output_tileable_variable(op.outputs[0])
+        return [f"{res_var_name} = {input_var_name}.mode({args_str})"]

maxframe/codegen/spe/dataframe/sort.py

@@ -16,9 +16,17 @@ from abc import abstractmethod
 from typing import Any, Dict, List
 
 from ....dataframe.core import DATAFRAME_TYPE
-from ....dataframe.sort import DataFrameSortIndex, DataFrameSortValues
+from ....dataframe.sort import DataFrameRank, DataFrameSortIndex, DataFrameSortValues
 from ....dataframe.sort.core import DataFrameSortOperator
 from ..core import SPECodeContext, SPEOperatorAdapter, register_op_adapter
+from ..utils import build_method_call_adapter
+
+DataFrameRankAdapter = build_method_call_adapter(
+    DataFrameRank,
+    "rank",
+    kw_keys=["axis", "method", "numeric_only", "na_option", "ascending", "pct"],
+    skip_none=True,
+)
 
 
 class _DataFrameSortAdapter(SPEOperatorAdapter):

maxframe/codegen/spe/dataframe/tests/test_reduction.py

@@ -19,6 +19,7 @@ from ...core import SPECodeContext
 from ..reduction import (
     DataFrameAggregateAdapter,
     DataFrameKurtosisAdapter,
+    DataFrameModeAdapter,
     DataFrameNuniqueAdapter,
     DataFrameReductionAdapter,
     DataFrameUniqueAdapter,
@@ -96,6 +97,18 @@ def test_median(df1):
     assert results == expected_results
 
 
+def test_mode(df1, s1):
+    df = df1.mode(dropna=False, numeric_only=True)
+    results = DataFrameModeAdapter().generate_code(df.op, SPECodeContext())
+    expected_results = ["var_1 = var_0.mode(axis=0, numeric_only=True, dropna=False)"]
+    assert results == expected_results
+
+    s = s1.mode()
+    results = DataFrameModeAdapter().generate_code(s.op, SPECodeContext())
+    expected_results = ["var_1 = var_0.mode(dropna=True)"]
+    assert results == expected_results
+
+
 def test_aggregate(df1):
     df = df1.agg(["sum", "mean"])
     context = SPECodeContext()

maxframe/codegen/spe/dataframe/tseries.py

@@ -14,9 +14,18 @@
 
 from typing import List
 
+from ....dataframe.tseries.between_time import DataFrameBetweenTime
 from ....dataframe.tseries.to_datetime import DataFrameToDatetime
 from ....utils import no_default
 from ..core import SPECodeContext, SPEOperatorAdapter, register_op_adapter
+from ..utils import build_method_call_adapter
+
+DataFrameBetweenTimeAdapter = build_method_call_adapter(
+    DataFrameBetweenTime,
+    "between_time",
+    kw_keys=["start_time", "end_time", "inclusive", "axis"],
+    skip_none=True,
+)
 
 
 @register_op_adapter(DataFrameToDatetime)

maxframe/codegen/spe/learn/contrib/lightgbm.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 import inspect
-from typing import List
+from typing import List, Optional
 
 from .....learn.contrib.lightgbm._predict import LGBMPredict
 from .....learn.contrib.lightgbm._train import LGBMTrain
@@ -23,11 +23,12 @@ from ...core import SPECodeContext, SPEOperatorAdapter, register_op_adapter
 
 
 class _LightGBMAdapter(SPEOperatorAdapter):
-    def generate_code(self, op: OperatorType, context: SPECodeContext) -> List[str]:
+    def generate_code(
+        self, op: OperatorType, context: SPECodeContext
+    ) -> Optional[List[str]]:
         context.register_import("lightgbm")
         context.register_import("pandas", "pd")
         context.register_import("numpy", "np")
-        return []
 
 
 @register_op_adapter(ToLGBMDataset)

maxframe/codegen/spe/tensor/datasource.py

@@ -142,6 +142,7 @@ class TensorOnesZerosLikeAdapter(SPEOperatorAdapter):
         ]
 
 
+@register_op_adapter(Scalar)
 class TensorScalarAdapter(SPEOperatorAdapter):
     def accepts(self, op: Scalar) -> EngineAcceptance:
         return EngineAcceptance.SUCCESSOR

maxframe/config/config.py

@@ -435,6 +435,9 @@ default_options.register_option(
 default_options.register_option(
     "session.quota_name", None, validator=is_null | is_string, remote=True
 )
+default_options.register_option(
+    "session.gu_quota_name", None, validator=is_null | is_string, remote=True
+)
 default_options.register_option(
     "session.region_id", None, validator=is_null | is_string, remote=True
 )

maxframe/conftest.py

@@ -61,6 +61,10 @@ def _get_account_env(test_config: ConfigParser, section_name: str) -> ODPS:
         tunnel_endpoint = test_config.get("odps", "tunnel_endpoint")
     except NoOptionError:
         tunnel_endpoint = None
+    try:
+        namespace = test_config.get("odps", "namespace")
+    except NoOptionError:
+        namespace = None
     return ODPS(
         access_id,
         secret_access_key,
@@ -68,6 +72,7 @@ def _get_account_env(test_config: ConfigParser, section_name: str) -> ODPS:
         endpoint,
         tunnel_endpoint=tunnel_endpoint,
         overwrite_global=False,
+        namespace=namespace,
     )
 
 
@@ -85,6 +90,7 @@ def _get_bearer_token_env(test_config: ConfigParser, section_name: str) -> ODPS:
         project=entry.project,
         endpoint=entry.endpoint,
         tunnel_endpoint=entry.tunnel_endpoint,
+        namespace=entry.namespace,
     )
 
 
@@ -97,6 +103,7 @@ def _enter_odps_envs(entry, drop_temp_tables=True):
         "ODPS_ENDPOINT",
         "RAY_ISOLATION_UT_ENV",
         "ODPS_TUNNEL_ENDPOINT",
+        "ODPS_NAMESPACE",
     ):
         if env_name in os.environ:
             stored_envs[env_name] = os.environ[env_name]
@@ -105,6 +112,8 @@ def _enter_odps_envs(entry, drop_temp_tables=True):
     os.environ["ODPS_BEARER_TOKEN"] = entry.account.token
     os.environ["ODPS_PROJECT_NAME"] = entry.project
     os.environ["ODPS_ENDPOINT"] = entry.endpoint
+    if entry.namespace:
+        os.environ["ODPS_NAMESPACE"] = entry.namespace
     os.environ["RAY_ISOLATION_UT_ENV"] = "UT"
     if entry.tunnel_endpoint:
         os.environ["ODPS_TUNNEL_ENDPOINT"] = entry.tunnel_endpoint
@@ -115,6 +124,7 @@ def _enter_odps_envs(entry, drop_temp_tables=True):
         os.environ.pop("ODPS_BEARER_TOKEN", None)
         os.environ.pop("ODPS_PROJECT_NAME", None)
         os.environ.pop("ODPS_ENDPOINT", None)
+        os.environ.pop("ODPS_NAMESPACE", None)
         os.environ.pop("ODPS_TUNNEL_ENDPOINT", None)
         os.environ.pop("RAY_ISOLATION_UT_ENV", None)
 

maxframe/core/base.py

@@ -126,12 +126,13 @@ class Base(Serializable):
 
     def to_kv(
         self,
-        exclude_fields: Tuple[str],
+        exclude_fields: Tuple[str] = None,
         accept_value_types: Optional[Tuple[Type]] = None,
     ):
         fields = self._FIELDS
         kv = {}
         no_value = object()
+        exclude_fields = exclude_fields or ()
         for name, field in fields.items():
             if name not in exclude_fields:
                 value = getattr(self, name, no_value)

maxframe/core/entity/tileables.py

@@ -318,6 +318,8 @@ class HasShapeTileableData(TileableData):
         try:
             return int(self.shape[0])
         except (IndexError, ValueError):  # pragma: no cover
+            if len(self.shape) == 0:
+                raise TypeError("len() of unsized object") from None
             return 0
 
     @property

maxframe/core/graph/entity.py

@@ -117,10 +117,11 @@ class SerializableGraph(Serializable):
     _successors = DictField("successors")
     _results = ListField("results")
     _graph_cls = StringField("graph_cls")
+    _extra_params = DictField("extra_params", default=None)
 
     @classmethod
     def from_graph(cls, graph: EntityGraph) -> "SerializableGraph":
-        return SerializableGraph(
+        kw = dict(
             _is_chunk=False,
             _fetch_nodes=[chunk for chunk in graph if chunk.is_fetch()],
             _nodes=graph._nodes,
@@ -129,6 +130,9 @@ class SerializableGraph(Serializable):
             _results=graph.results,
             _graph_cls=extract_class_name(type(graph)),
         )
+        if hasattr(graph, "extra_params"):
+            kw["_extra_params"] = graph.extra_params
+        return SerializableGraph(**kw)
 
     def to_graph(self) -> EntityGraph:
         graph_cls = (
@@ -140,6 +144,8 @@ class SerializableGraph(Serializable):
         graph._nodes.update(self._nodes)
         graph._predecessors.update(self._predecessors)
         graph._successors.update(self._successors)
+        if self._extra_params:
+            graph.extra_params = self._extra_params
         return graph
 
 

maxframe/core/mode.py

@@ -42,6 +42,10 @@ def is_build_mode():
     return bool(getattr(_internal_mode, "build", False))
 
 
+def is_mock_mode():
+    return bool(getattr(_internal_mode, "mock", False))
+
+
 class _EnterModeFuncWrapper:
     def __init__(self, mode_name_to_value):
         self.mode_name_to_value = mode_name_to_value
@@ -86,10 +90,11 @@ class _EnterModeFuncWrapper:
         return _inner
 
 
-def enter_mode(kernel=None, build=None):
+def enter_mode(kernel=None, build=None, mock=None):
     mode_name_to_value = {
         "kernel": kernel,
         "build": build,
+        "mock": mock,
     }
     mode_name_to_value = {k: v for k, v in mode_name_to_value.items() if v is not None}
 

maxframe/dataframe/__init__.py

@@ -32,6 +32,7 @@ from . import (
     window,
 )
 from .datasource.date_range import date_range
+from .datasource.direct import read_clipboard
 from .datasource.from_index import series_from_index
 from .datasource.from_records import from_records
 from .datasource.from_tensor import dataframe_from_tensor, series_from_tensor
@@ -40,7 +41,6 @@ from .datasource.read_odps_query import read_odps_query
 from .datasource.read_odps_table import read_odps_table
 from .datasource.read_parquet import read_parquet
 from .datastore.to_odps import to_odps_table
-from .groupby import NamedAgg
 from .initializer import DataFrame, Index, Series, read_pandas
 from .merge import concat, merge
 from .misc.cut import cut
@@ -49,7 +49,7 @@ from .misc.get_dummies import get_dummies
 from .misc.qcut import qcut
 from .misc.to_numeric import to_numeric
 from .missing import isna, isnull, notna, notnull
-from .reduction import CustomReduction, unique
+from .reduction import CustomReduction, NamedAgg, unique
 from .reshape.melt import melt
 from .reshape.pivot import pivot
 from .reshape.pivot_table import pivot_table

maxframe/dataframe/arithmetic/__init__.py

@@ -53,6 +53,8 @@ from .less_equal import DataFrameLessEqual, le
 from .log import DataFrameLog
 from .log2 import DataFrameLog2
 from .log10 import DataFrameLog10
+from .maximum import DataFrameMaximum
+from .minimum import DataFrameMinimum
 from .mod import DataFrameMod, mod, rmod
 from .multiply import DataFrameMul, mul, rmul
 from .negative import DataFrameNegative, negative
@@ -250,6 +252,8 @@ def _install():
         DataFramePower,
         DataFrameSubtract,
         DataFrameTrueDiv,
+        DataFrameMaximum,
+        DataFrameMinimum,
     ]
     for ufunc_op in ufunc_ops:
         register_tensor_ufunc(ufunc_op)

maxframe/dataframe/arithmetic/maximum.py

@@ -0,0 +1,33 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from ... import opcodes
+from ...utils import classproperty
+from .core import DataFrameBinopUfunc
+
+
+class DataFrameMaximum(DataFrameBinopUfunc):
+    _op_type_ = opcodes.MAXIMUM
+
+    @classproperty
+    def _operator(self):
+        return np.maximum
+
+    @classproperty
+    def tensor_op_type(self):
+        from ...tensor.arithmetic import TensorMaximum
+
+        return TensorMaximum

maxframe/dataframe/arithmetic/minimum.py

@@ -0,0 +1,33 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from ... import opcodes
+from ...utils import classproperty
+from .core import DataFrameBinopUfunc
+
+
+class DataFrameMinimum(DataFrameBinopUfunc):
+    _op_type_ = opcodes.MINIMUM
+
+    @classproperty
+    def _operator(self):
+        return np.minimum
+
+    @classproperty
+    def tensor_op_type(self):
+        from ...tensor.arithmetic import TensorMinimum
+
+        return TensorMinimum