maxframe 1.3.0__cp311-cp311-win32.whl → 2.0.0b1__cp311-cp311-win32.whl

maxframe/codegen/spe/dataframe/accessors/base.py

@@ -0,0 +1,53 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+
+from .....dataframe.accessors.datetime_ import SeriesDatetimeMethod
+from .....dataframe.accessors.string_ import SeriesStringMethod
+from ...core import SPECodeContext, SPEOperatorAdapter, register_op_adapter
+
+
+@register_op_adapter(SeriesDatetimeMethod)
+class SeriesDatetimeMethodAdapter(SPEOperatorAdapter):
+    def generate_code(
+        self, op: SeriesDatetimeMethod, context: SPECodeContext
+    ) -> List[str]:
+        inst_var_name = context.get_input_tileable_variable(op.inputs[0])
+        res_var_name = context.get_output_tileable_variable(op.outputs[0])
+        method_str = f"{inst_var_name}.dt.{op.method}"
+        if op.is_property:
+            return [f"{res_var_name} = {method_str}"]
+        else:
+            args_list = self._translate_call_args(
+                context, *op.method_args, **op.method_kwargs
+            )
+            args = ", ".join(args_list)
+            return [f"{res_var_name} = {method_str}({args})"]
+
+
+@register_op_adapter(SeriesStringMethod)
+class SeriesStringMethodAdapter(SPEOperatorAdapter):
+    def generate_code(
+        self, op: SeriesStringMethod, context: SPECodeContext
+    ) -> List[str]:
+        inst_var_name = context.get_input_tileable_variable(op.inputs[0])
+        res_var_name = context.get_output_tileable_variable(op.outputs[0])
+        method_str = f"{inst_var_name}.str.{op.method}"
+
+        args_list = self._translate_call_args(
+            context, *op.method_args, **op.method_kwargs
+        )
+        args = ", ".join(args_list)
+        return [f"{res_var_name} = {method_str}({args})"]

maxframe/codegen/spe/dataframe/accessors/dict_.py

@@ -0,0 +1,194 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+
+from .....dataframe.accessors.dict_.contains import SeriesDictContainsOperator
+from .....dataframe.accessors.dict_.getitem import SeriesDictGetItemOperator
+from .....dataframe.accessors.dict_.length import SeriesDictLengthOperator
+from .....dataframe.accessors.dict_.remove import SeriesDictRemoveOperator
+from .....dataframe.accessors.dict_.setitem import SeriesDictSetItemOperator
+from ...core import SPECodeContext, SPEOperatorAdapter, register_op_adapter
+
+_get_template = """
+def _inner_get(data):
+    found = False
+    for tup in data:
+        if tup[0] == {key_var}:
+            found = True
+            return tup[1]
+    if not found:
+        if {ignore_key_error_var}:
+            return {default_value_var}
+        else:
+            raise KeyError({key_var})
+
+{output_var} = {input_var}.map(_inner_get, na_action="ignore").astype({output_dtype_var})
+{output_var}.name = {key_var}
+"""
+
+_set_template = """
+def _inner_set(row):
+    found = False
+    value = list()
+    for tup in row:
+        if tup[0] == {key_var}:
+            value.append((tup[0], {value_var}))
+            found = True
+        else:
+            value.append(tup)
+    if not found:
+        value.append(({key_var}, {value_var}))
+    return value
+
+{output_var} = {input_var}.map(_inner_set, na_action="ignore").astype({output_dtype_var})
+"""
+
+_length_template = """
+{output_var} = {input_var}.map(len, na_action="ignore").astype({output_dtype_var})
+{output_var}.name = None
+"""
+
+
+_remove_template = """
+def _inner_remove(value):
+    row = list()
+    found = False
+    for tup in value:
+        if tup[0] == {key_var}:
+            found = True
+        else:
+            row.append(tup)
+    if not found and not {ignore_key_error_var}:
+        raise KeyError({key_var})
+    return row
+
+{output_var} = {input_var}.map(_inner_remove, na_action="ignore").astype({output_dtype_var})
+"""
+
+_contains_template = """
+{output_var} = {input_var}.map(lambda x: any({key_var} in tup[0] for tup in x), na_action="ignore").astype({output_dtype_var})
+{output_var}.name = None
+"""
+
+
+@register_op_adapter(SeriesDictGetItemOperator)
+class SeriesDictGetItemOperatorAdapter(SPEOperatorAdapter):
+    def generate_code(
+        self, op: SeriesDictGetItemOperator, context: SPECodeContext
+    ) -> List[str]:
+        input_var = context.get_input_tileable_variable(op.inputs[0])
+        output_var = context.get_output_tileable_variable(op.outputs[0])
+        key_var = self.translate_var(context, op.query_key)
+        output_name_var = self.translate_var(context, op.outputs[0].name)
+        default_value_var = self.translate_var(context, op.default_value)
+        ignore_key_error_var = self.translate_var(context, op.ignore_key_error)
+        output_dtype_var = context.register_operator_constants(op.outputs[0].dtype)
+        return [
+            _get_template.format(
+                input_var=input_var,
+                output_var=output_var,
+                key_var=key_var,
+                default_value_var=default_value_var,
+                ignore_key_error_var=ignore_key_error_var,
+                output_name_var=output_name_var,
+                output_dtype_var=output_dtype_var,
+            )
+        ]
+
+
+@register_op_adapter(SeriesDictSetItemOperator)
+class SeriesDictSetItemOperatorAdapter(SPEOperatorAdapter):
+    def generate_code(
+        self, op: SeriesDictSetItemOperator, context: SPECodeContext
+    ) -> List[str]:
+        input_var = context.get_input_tileable_variable(op.inputs[0])
+        output_var = context.get_output_tileable_variable(op.outputs[0])
+        key_var = self.translate_var(context, op.query_key)
+        value_var = self.translate_var(context, op.value)
+        output_name_var = self.translate_var(context, op.outputs[0].name)
+        output_dtype_var = context.register_operator_constants(op.outputs[0].dtype)
+        return [
+            _set_template.format(
+                input_var=input_var,
+                output_var=output_var,
+                key_var=key_var,
+                value_var=value_var,
+                output_name_var=output_name_var,
+                output_dtype_var=output_dtype_var,
+            )
+        ]
+
+
+@register_op_adapter(SeriesDictLengthOperator)
+class SeriesDictLengthOperatorAdapter(SPEOperatorAdapter):
+    def generate_code(
+        self, op: SeriesDictLengthOperator, context: SPECodeContext
+    ) -> List[str]:
+        input_var = context.get_input_tileable_variable(op.inputs[0])
+        output_var = context.get_output_tileable_variable(op.outputs[0])
+        output_name_var = self.translate_var(context, op.outputs[0].name)
+        output_dtype_var = context.register_operator_constants(op.outputs[0].dtype)
+        return [
+            _length_template.format(
+                input_var=input_var,
+                output_var=output_var,
+                output_name_var=output_name_var,
+                output_dtype_var=output_dtype_var,
+            )
+        ]
+
+
+@register_op_adapter(SeriesDictRemoveOperator)
+class SeriesDictRemoveOperatorAdapter(SPEOperatorAdapter):
+    def generate_code(
+        self, op: SeriesDictRemoveOperator, context: SPECodeContext
+    ) -> List[str]:
+        input_var = context.get_input_tileable_variable(op.inputs[0])
+        output_var = context.get_output_tileable_variable(op.outputs[0])
+        output_name_var = self.translate_var(context, op.outputs[0].name)
+        output_dtype_var = context.register_operator_constants(op.outputs[0].dtype)
+        key_var = self.translate_var(context, op.query_key)
+        ignore_key_error_var = self.translate_var(context, op.ignore_key_error)
+        return [
+            _remove_template.format(
+                input_var=input_var,
+                output_var=output_var,
+                key_var=key_var,
+                ignore_key_error_var=ignore_key_error_var,
+                output_name_var=output_name_var,
+                output_dtype_var=output_dtype_var,
+            )
+        ]
+
+
+@register_op_adapter(SeriesDictContainsOperator)
+class SeriesDictContainsOperatorAdapter(SPEOperatorAdapter):
+    def generate_code(
+        self, op: SeriesDictContainsOperator, context: SPECodeContext
+    ) -> List[str]:
+        input_var = context.get_input_tileable_variable(op.inputs[0])
+        output_var = context.get_output_tileable_variable(op.outputs[0])
+        output_name_var = self.translate_var(context, op.outputs[0].name)
+        output_dtype_var = context.register_operator_constants(op.outputs[0].dtype)
+        key_var = self.translate_var(context, op.query_key)
+        return [
+            _contains_template.format(
+                input_var=input_var,
+                output_var=output_var,
+                key_var=key_var,
+                output_name_var=output_name_var,
+                output_dtype_var=output_dtype_var,
+            )
+        ]

maxframe/codegen/spe/dataframe/accessors/list_.py

@@ -0,0 +1,80 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+
+from .....dataframe.accessors.list_.getitem import SeriesListGetItemOperator
+from .....dataframe.accessors.list_.length import SeriesListLengthOperator
+from ...core import SPECodeContext, SPEOperatorAdapter, register_op_adapter
+
+_get_template = """
+def _inner_get(data):
+    try:
+        return data[{index_var}]
+    except IndexError:
+        if {ignore_index_error_var}:
+            return None
+        else:
+            raise
+
+{output_var} = {input_var}.map(_inner_get, na_action="ignore").astype({output_dtype_var})
+{output_var}.name = None
+"""
+
+_length_template = """
+{output_var} = {input_var}.map(len, na_action="ignore").astype({output_dtype_var})
+{output_var}.name = None
+"""
+
+
+@register_op_adapter(SeriesListGetItemOperator)
+class SeriesListGetItemOperatorAdapter(SPEOperatorAdapter):
+    def generate_code(
+        self, op: SeriesListGetItemOperator, context: SPECodeContext
+    ) -> List[str]:
+        input_var = context.get_input_tileable_variable(op.inputs[0])
+        output_var = context.get_output_tileable_variable(op.outputs[0])
+        index_var = self.translate_var(context, op.query_index)
+        output_name_var = self.translate_var(context, op.outputs[0].name)
+        ignore_index_error_var = self.translate_var(context, op.ignore_index_error)
+        output_dtype_var = context.register_operator_constants(op.outputs[0].dtype)
+        return [
+            _get_template.format(
+                input_var=input_var,
+                output_var=output_var,
+                index_var=index_var,
+                ignore_index_error_var=ignore_index_error_var,
+                output_name_var=output_name_var,
+                output_dtype_var=output_dtype_var,
+            )
+        ]
+
+
+@register_op_adapter(SeriesListLengthOperator)
+class SeriesListLengthOperatorAdapter(SPEOperatorAdapter):
+    def generate_code(
+        self, op: SeriesListLengthOperator, context: SPECodeContext
+    ) -> List[str]:
+        input_var = context.get_input_tileable_variable(op.inputs[0])
+        output_var = context.get_output_tileable_variable(op.outputs[0])
+        output_name_var = self.translate_var(context, op.outputs[0].name)
+        output_dtype_var = context.register_operator_constants(op.outputs[0].dtype)
+        return [
+            _length_template.format(
+                input_var=input_var,
+                output_var=output_var,
+                output_name_var=output_name_var,
+                output_dtype_var=output_dtype_var,
+            )
+        ]

maxframe/codegen/spe/dataframe/arithmetic.py

@@ -0,0 +1,84 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Dict, List, Type
+
+from ....core import ENTITY_TYPE
+from ....dataframe.arithmetic import (
+    DataFrameAnd,
+    DataFrameAround,
+    DataFrameOr,
+    DataFrameXor,
+)
+from ....dataframe.arithmetic.core import DataFrameBinOp, DataFrameUnaryOp
+from ....dataframe.core import SERIES_TYPE
+from ..core import SPECodeContext, SPEOperatorAdapter, register_op_adapter
+
+
+@register_op_adapter(DataFrameUnaryOp)
+class DataFrameUnaryFuncAdapter(SPEOperatorAdapter):
+    def generate_code(self, op: DataFrameUnaryOp, context: SPECodeContext) -> List[str]:
+        context.register_import("numpy", "np")
+
+        inp_var = context.get_input_tileable_variable(op.inputs[0])
+        res_var_name = context.get_output_tileable_variable(op.outputs[0])
+        func = getattr(op, "_func_name")
+        if isinstance(op, DataFrameAround):
+            decimals_var = self.translate_var(context, op.decimals)
+            return [f"{res_var_name} = {inp_var}.round({decimals_var})"]
+        else:
+            return [f"{res_var_name} = np.{func}({inp_var})"]
+
+
+@register_op_adapter(DataFrameBinOp)
+class DataFrameBinOpAdapter(SPEOperatorAdapter):
+    def generate_code(self, op: DataFrameBinOp, context: SPECodeContext) -> List[str]:
+        use_reversed = not isinstance(op.lhs, ENTITY_TYPE)
+        kw = {}
+        for arg in ("axis", "level", "fill_value"):
+            if getattr(op, arg, None) is not None:
+                kw[arg] = getattr(op, arg)
+        if isinstance(op.inputs[0], SERIES_TYPE) and "axis" in kw:
+            kw.pop("axis")
+
+        if use_reversed:
+            func = getattr(op, "_rfunc_name")
+            inst, arg = op.rhs, op.lhs
+        else:
+            func = getattr(op, "_func_name")
+            inst, arg = op.lhs, op.rhs
+
+        inst_var_name = context.get_input_tileable_variable(inst)
+        args_list = self._translate_call_args(context, arg, **kw)
+        args = ", ".join(args_list)
+        res_var_name = context.get_output_tileable_variable(op.outputs[0])
+        return [f"{res_var_name} = {inst_var_name}.{func}({args})"]
+
+
+@register_op_adapter([DataFrameAnd, DataFrameOr, DataFrameXor])
+class DataFrameBitwiseBinOpAdapter(SPEOperatorAdapter):
+    _python_op: Dict[Type[DataFrameBinOp], str] = {
+        DataFrameAnd: "&",
+        DataFrameOr: "|",
+        DataFrameXor: "^",
+    }
+
+    def generate_code(self, op: DataFrameBinOp, context: SPECodeContext) -> List[str]:
+        lhs, rhs = (
+            (op.lhs, op.rhs) if isinstance(op.lhs, ENTITY_TYPE) else (op.rhs, op.lhs)
+        )
+        left_var = context.get_input_tileable_variable(lhs)
+        res_var = context.get_output_tileable_variable(op.outputs[0])
+        right_var = self._translate_call_args(context, rhs)[0]
+        return [f"{res_var} = {left_var} {self._python_op[type(op)]} {right_var}"]

maxframe/codegen/spe/dataframe/datasource.py

@@ -0,0 +1,181 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List, Optional
+
+import numpy as np
+import pandas as pd
+import pyarrow as pa
+from odps import ODPS
+
+from ....dataframe.datasource.dataframe import DataFrameDataSource
+from ....dataframe.datasource.date_range import DataFrameDateRange
+from ....dataframe.datasource.from_records import DataFrameFromRecords
+from ....dataframe.datasource.from_tensor import DataFrameFromTensor, SeriesFromTensor
+from ....dataframe.datasource.index import IndexDataSource
+from ....dataframe.datasource.read_odps_table import DataFrameReadODPSTable
+from ....io.odpsio import ODPSTableIO, arrow_to_pandas
+from ....protocol import DataFrameTableMeta
+from ..core import SPECodeContext, SPEOperatorAdapter, register_op_adapter
+
+
+@register_op_adapter(DataFrameDataSource)
+class DataFrameDataSourceAdapter(SPEOperatorAdapter):
+    def generate_code(
+        self, op: DataFrameDataSource, context: SPECodeContext
+    ) -> List[str]:
+        data_var = context.register_operator_constants(op.data)
+        res_var_name = context.get_output_tileable_variable(op.outputs[0])
+        return [f"{res_var_name} = {data_var}"]
+
+
+@register_op_adapter(DataFrameDateRange)
+class DataFrameDateRangeAdapter(SPEOperatorAdapter):
+    def generate_code(
+        self, op: DataFrameDateRange, context: SPECodeContext
+    ) -> List[str]:
+        context.register_import("pandas", "pd")
+
+        kw_keys = [
+            "start",
+            "end",
+            "periods",
+            "freq",
+            "tz",
+            "normalize",
+            "name",
+            "inclusive",
+        ]
+        args = self.generate_call_args_with_attributes(
+            op, context, kw_keys=kw_keys, skip_none=True
+        )
+        res_var_name = context.get_output_tileable_variable(op.outputs[0])
+        return [f"{res_var_name} = pd.date_range({args})"]
+
+
+@register_op_adapter(DataFrameFromRecords)
+class DataFrameFromRecordsAdapter(SPEOperatorAdapter):
+    def generate_code(
+        self, op: DataFrameFromRecords, context: SPECodeContext
+    ) -> List[str]:
+        context.register_import("pandas", "pd")
+
+        args = self.generate_call_args_with_attributes(
+            op,
+            context,
+            "input",
+            kw_keys=["columns", "exclude", "coerce_float", "nrows"],
+            skip_none=True,
+        )
+        res_var_name = context.get_output_tileable_variable(op.outputs[0])
+        return [f"{res_var_name} = pd.DataFrame.from_records({args})"]
+
+
+@register_op_adapter(DataFrameFromTensor)
+class DataFrameFromTensorAdapter(SPEOperatorAdapter):
+    def generate_code(
+        self, op: DataFrameFromTensor, context: SPECodeContext
+    ) -> List[str]:
+        context.register_import("pandas", "pd")
+
+        args = self.generate_call_args_with_attributes(
+            op, context, "input", kw_keys=["index", "columns"], skip_none=True
+        )
+        res_var_name = context.get_output_tileable_variable(op.outputs[0])
+        return [f"{res_var_name} = pd.DataFrame({args})"]
+
+
+@register_op_adapter(DataFrameReadODPSTable)
+class DataFrameReadODPSTableAdapter(SPEOperatorAdapter):
+    @staticmethod
+    def _read_as_pandas(
+        table_name: str,
+        partitions: Optional[List[str]],
+        columns: Optional[List[str]],
+        index_columns: Optional[List[str]],
+        append_partitions: bool,
+        table_meta: DataFrameTableMeta,
+    ):
+        o = ODPS.from_environments() or ODPS.from_global()
+
+        if columns is not None:
+            all_cols = (index_columns or []) + (columns or [])
+        else:
+            all_cols = None
+
+        with ODPSTableIO(o).open_reader(
+            table_name, partitions, all_cols, partition_columns=append_partitions
+        ) as reader:
+            arrow_result = reader.read_all()
+        if not index_columns:
+            # need prepending an index column
+            arrow_result = arrow_result.add_column(
+                0,
+                table_meta.table_index_column_names[0],
+                pa.array(np.arange(len(arrow_result))),
+            )
+        else:
+            name_mapping = dict(zip(index_columns, table_meta.table_index_column_names))
+            new_names = [name_mapping.get(c, c) for c in arrow_result.column_names]
+            arrow_result = arrow_result.rename_columns(new_names)
+
+        res = arrow_to_pandas(arrow_result, table_meta)
+        if not index_columns:
+            res.index = pd.RangeIndex(len(arrow_result))
+        return res
+
+    def generate_code(
+        self, op: DataFrameReadODPSTable, context: SPECodeContext
+    ) -> List[str]:
+        table_meta = context.get_pandas_data_table_meta(op.outputs[0])
+
+        cls_name = type(self).__name__
+        context.register_import(__name__, from_item=cls_name)
+        res_var_name = context.get_output_tileable_variable(op.outputs[0])
+        args = self._translate_call_args(
+            context,
+            op.table_name,
+            op.partitions,
+            op.columns,
+            op.index_columns,
+            op.append_partitions,
+            table_meta,
+        )
+        return [f"{res_var_name} = {cls_name}._read_as_pandas({', '.join(args)})"]
+
+
+@register_op_adapter(SeriesFromTensor)
+class SeriesFromTensorAdapter(SPEOperatorAdapter):
+    def generate_code(self, op: SeriesFromTensor, context: SPECodeContext) -> List[str]:
+        context.register_import("pandas", "pd")
+        tensor_var_name = context.get_input_tileable_variable(op.input)
+        call_args = self._translate_call_args(
+            context,
+            index=op.index,
+            name=op.outputs[0].name,
+            dtype=op.outputs[0].dtype,
+        )
+        res_var_name = context.get_output_tileable_variable(op.outputs[0])
+        return [
+            f"{res_var_name} = pd.Series({tensor_var_name}, {', '.join(call_args)})"
+        ]
+
+
+@register_op_adapter(IndexDataSource)
+class IndexDataSourceAdapter(SPEOperatorAdapter):
+    def generate_code(self, op: IndexDataSource, context: SPECodeContext) -> List[str]:
+        context.register_import("pandas")
+        data_var = self.translate_var(context, op.data)
+        res_var = context.get_output_tileable_variable(op.outputs[0])
+        return [f"{res_var} = {data_var}"]