maxframe 2.0.0b2__cp39-cp39-macosx_10_9_universal2.whl → 2.2.0__cp39-cp39-macosx_10_9_universal2.whl

maxframe/dataframe/extensions/apply_chunk.py

@@ -26,9 +26,10 @@ from ...serialization.serializables import (
     Int32Field,
     TupleField,
 )
+from ...typing_ import TileableType
 from ...udf import BuiltinFunction, MarkedFunction
 from ...utils import copy_if_possible, make_dtype, make_dtypes
-from ..core import DATAFRAME_TYPE, DataFrame, IndexValue, Series
+from ..core import DATAFRAME_TYPE, INDEX_TYPE, DataFrame, IndexValue, Series
 from ..operators import DataFrameOperator, DataFrameOperatorMixin
 from ..utils import (
     InferredDataFrameMeta,
@@ -43,7 +44,7 @@ from ..utils import (
 
 class DataFrameApplyChunk(DataFrameOperator, DataFrameOperatorMixin):
     _op_type_ = opcodes.APPLY_CHUNK
-    _legacy_name = "DataFrameApplyChunkOperator"
+    _legacy_name = "DataFrameApplyChunkOperator"  # since v2.0.0
 
     func = FunctionField("func")
     batch_rows = Int32Field("batch_rows", default=None)
@@ -60,16 +61,26 @@ class DataFrameApplyChunk(DataFrameOperator, DataFrameOperatorMixin):
     def has_custom_code(self) -> bool:
         return not isinstance(self.func, BuiltinFunction)
 
+    def check_inputs(self, inputs: List[TileableType]):
+        # for apply_chunk we allow called on non-deterministic tileables
+        pass
+
     def _call_dataframe(self, df, dtypes, dtype, name, index_value, element_wise):
         # return dataframe
         if self.output_types[0] == OutputType.dataframe:
             dtypes = make_dtypes(dtypes)
+            if dtypes is not None:
+                shape = df.shape if element_wise else (np.nan, len(dtypes))
+                cols_value = parse_index(dtypes.index, store_data=True)
+            else:
+                shape = (np.nan, np.nan)
+                cols_value = None
             # apply_chunk will use generate new range index for results
             return self.new_dataframe(
                 [df],
-                shape=df.shape if element_wise else (np.nan, len(dtypes)),
+                shape=shape,
                 index_value=index_value,
-                columns_value=parse_index(dtypes.index, store_data=True),
+                columns_value=cols_value,
                 dtypes=dtypes,
             )
 
@@ -106,11 +117,17 @@ class DataFrameApplyChunk(DataFrameOperator, DataFrameOperatorMixin):
         name: Any = None,
         output_type=None,
         index=None,
+        skip_infer=False,
     ):
         args = self.args or ()
         kwargs = self.kwargs or {}
         # if not dtypes and not skip_infer:
-        packed_func = get_packed_func(df_or_series, self.func, *args, **kwargs)
+        try:
+            packed_func = get_packed_func(df_or_series, self.func, *args, **kwargs)
+        except:
+            if not skip_infer:
+                raise
+            packed_func = self.func
 
         # if skip_infer, directly build a frame
         if self.output_types and self.output_types[0] == OutputType.df_or_series:
@@ -125,13 +142,15 @@ class DataFrameApplyChunk(DataFrameOperator, DataFrameOperatorMixin):
             dtype=dtype,
             name=name,
             index=index,
+            skip_infer=skip_infer,
         )
 
         if inferred_meta.index_value is None:
             inferred_meta.index_value = parse_index(
                 None, (df_or_series.key, df_or_series.index_value.key, self.func)
             )
-        inferred_meta.check_absence("output_type", "dtypes", "dtype")
+        if not skip_infer:
+            inferred_meta.check_absence("output_type", "dtypes", "dtype")
 
         if isinstance(df_or_series, DATAFRAME_TYPE):
             return self._call_dataframe(
@@ -163,6 +182,7 @@ class DataFrameApplyChunk(DataFrameOperator, DataFrameOperatorMixin):
         name: Any = None,
         index: Union[pd.Index, IndexValue] = None,
         elementwise: bool = None,
+        skip_infer: bool = False,
         **kwargs,
     ) -> InferredDataFrameMeta:
         inferred_meta = infer_dataframe_return_value(
@@ -174,7 +194,10 @@ class DataFrameApplyChunk(DataFrameOperator, DataFrameOperatorMixin):
             name=name,
             index=index,
             elementwise=elementwise,
+            skip_infer=skip_infer,
         )
+        if skip_infer:
+            return inferred_meta
 
         # merge specified and inferred index, dtypes, output_type
         # elementwise used to decide shape
@@ -186,6 +209,8 @@ class DataFrameApplyChunk(DataFrameOperator, DataFrameOperatorMixin):
         if self.output_types:
             inferred_meta.output_type = self.output_types[0]
         inferred_meta.dtypes = dtypes if dtypes is not None else inferred_meta.dtypes
+        if isinstance(index, INDEX_TYPE):
+            index = index.index_value
         if index is not None:
             inferred_meta.index_value = (
                 parse_index(index)
@@ -458,6 +483,7 @@ def df_apply_chunk(
         name=name,
         index=index,
         output_type=output_type,
+        skip_infer=skip_infer,
     )
 
 

maxframe/dataframe/extensions/cartesian_chunk.py

@@ -0,0 +1,153 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+
+import numpy as np
+import pandas as pd
+
+from ... import opcodes
+from ...core import EntityData, OutputType
+from ...serialization.serializables import (
+    DictField,
+    FunctionField,
+    KeyField,
+    TupleField,
+)
+from ...udf import BuiltinFunction
+from ...utils import quiet_stdio
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+from ..utils import (
+    build_df,
+    build_empty_df,
+    build_series,
+    parse_index,
+    validate_output_types,
+)
+
+
+class DataFrameCartesianChunk(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.CARTESIAN_CHUNK
+
+    left = KeyField("left")
+    right = KeyField("right")
+    func = FunctionField("func")
+    args = TupleField("args")
+    kwargs = DictField("kwargs")
+
+    def __init__(self, output_types=None, **kw):
+        super().__init__(_output_types=output_types, **kw)
+        if self.memory_scale is None:
+            self.memory_scale = 2.0
+
+    @classmethod
+    def _set_inputs(cls, op: "DataFrameCartesianChunk", inputs: List[EntityData]):
+        super()._set_inputs(op, inputs)
+        op.left, op.right = op.inputs[:2]
+
+    @staticmethod
+    def _build_test_obj(obj):
+        return (
+            build_df(obj, size=2)
+            if obj.ndim == 2
+            else build_series(obj, size=2, name=obj.name)
+        )
+
+    def has_custom_code(self) -> bool:
+        return not isinstance(self.func, BuiltinFunction)
+
+    def __call__(self, left, right, index=None, dtypes=None):
+        test_left = self._build_test_obj(left)
+        test_right = self._build_test_obj(right)
+        output_type = self._output_types[0] if self._output_types else None
+
+        if output_type == OutputType.df_or_series:
+            return self.new_df_or_series([left, right])
+
+        # try run to infer meta
+        try:
+            with np.errstate(all="ignore"), quiet_stdio():
+                obj = self.func(test_left, test_right, *self.args, **self.kwargs)
+        except:  # noqa: E722  # nosec  # pylint: disable=bare-except
+            if output_type == OutputType.series:
+                obj = pd.Series([], dtype=np.dtype(object))
+            elif output_type == OutputType.dataframe and dtypes is not None:
+                obj = build_empty_df(dtypes)
+            else:
+                raise TypeError(
+                    "Cannot determine `output_type`, "
+                    "you have to specify it as `dataframe` or `series`, "
+                    "for dataframe, `dtypes` is required as well "
+                    "if output_type='dataframe'"
+                )
+
+        if getattr(obj, "ndim", 0) == 1 or output_type == OutputType.series:
+            shape = self.kwargs.pop("shape", (np.nan,))
+            if index is None:
+                index = obj.index
+            index_value = parse_index(
+                index, left, right, self.func, self.args, self.kwargs
+            )
+            return self.new_series(
+                [left, right],
+                dtype=obj.dtype,
+                shape=shape,
+                index_value=index_value,
+                name=obj.name,
+            )
+        else:
+            dtypes = dtypes if dtypes is not None else obj.dtypes
+            # dataframe
+            shape = (np.nan, len(dtypes))
+            columns_value = parse_index(dtypes.index, store_data=True)
+            if index is None:
+                index = obj.index
+            index_value = parse_index(
+                index, left, right, self.func, self.args, self.kwargs
+            )
+            return self.new_dataframe(
+                [left, right],
+                shape=shape,
+                dtypes=dtypes,
+                index_value=index_value,
+                columns_value=columns_value,
+            )
+
+
+def cartesian_chunk(left, right, func, skip_infer=False, args=(), **kwargs):
+    output_type = kwargs.pop("output_type", None)
+    output_types = kwargs.pop("output_types", None)
+    object_type = kwargs.pop("object_type", None)
+    output_types = validate_output_types(
+        output_type=output_type, output_types=output_types, object_type=object_type
+    )
+    output_type = output_types[0] if output_types else None
+    if output_type:
+        output_types = [output_type]
+    elif skip_infer:
+        output_types = [OutputType.df_or_series]
+    index = kwargs.pop("index", None)
+    dtypes = kwargs.pop("dtypes", None)
+    memory_scale = kwargs.pop("memory_scale", None)
+
+    op = DataFrameCartesianChunk(
+        left=left,
+        right=right,
+        func=func,
+        args=args,
+        kwargs=kwargs,
+        output_types=output_types,
+        memory_scale=memory_scale,
+    )
+    return op(left, right, index=index, dtypes=dtypes)

maxframe/dataframe/extensions/collect_kv.py

@@ -0,0 +1,126 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+import pandas as pd
+
+from ... import opcodes
+from ...serialization.serializables import AnyField, StringField
+from ...utils import no_default
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+from ..utils import make_column_list
+
+
+class DataFrameCollectKv(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.COLLECT_KV
+
+    columns = AnyField("columns", default=None)
+    kv_delim = StringField("kv_delim", default=None)
+    item_delim = StringField("item_delim", default=None)
+    kv_col = StringField("kv_col", default=None)
+
+    def __call__(self, df):
+        if self.columns is None:
+            cols = list(df.dtypes.index)
+        else:
+            cols = self.columns if isinstance(self.columns, list) else [self.columns]
+        new_dtypes = df.dtypes.drop(cols, errors="ignore")
+        new_dtypes = pd.concat(
+            [new_dtypes, pd.Series([np.dtype("object")], index=[self.kv_col])]
+        )
+        shape = (df.shape[0], len(new_dtypes))
+        return self.new_dataframe(
+            [df],
+            shape=shape,
+            dtypes=new_dtypes,
+            index_value=df.index_value,
+            columns_value=new_dtypes.index,
+        )
+
+
+def collect_kv(
+    data,
+    columns=None,
+    kv_delim="=",
+    item_delim=",",
+    kv_col="kv_col",
+):
+    """
+    Merge values in specified columns into a key-value represented column.
+
+    Parameters
+    ----------
+    columns : list, default None
+        The columns to be merged.
+    kv_delim : str, default '='
+        Delimiter between key and value.
+    item_delim : str, default ','
+        Delimiter between key-value pairs.
+    kv_col : str, default 'kv_col'
+        Name of the new key-value column
+
+    Returns
+    -------
+    DataFrame
+        converted data frame
+
+    See Also
+    --------
+    DataFrame.mf.extract_kv
+
+    Examples
+    -------
+    >>> import maxframe.dataframe as md
+
+    >>> df = md.DataFrame({"name": ["name1", "name2", "name3", "name4", "name5"],
+    ...                    "k1": [1.0, NaN, 7.1, NaN, NaN],
+    ...                    "k2": [3.0, 3.0, NaN, 1.2, 1.0],
+    ...                    "k3": [NaN, 5.1, NaN, 1.5, NaN],
+    ...                    "k5": [10.0, NaN, NaN, NaN, NaN,],
+    ...                    "k7": [NaN, NaN, 8.2, NaN, NaN, ],
+    ...                    "k9": [NaN, NaN, NaN, NaN, 1.1]})
+    >>> df.execute()
+       name   k1   k2   k3   k5    k7   k9
+    0  name1  1.0  3.0  NaN  10.0  NaN  NaN
+    1  name2  NaN  3.0  5.1  NaN   NaN  NaN
+    2  name3  7.1  NaN  NaN  NaN   8.2  NaN
+    3  name4  NaN  1.2  1.5  NaN   NaN  NaN
+    4  name5  NaN  1.0  NaN  NaN   NaN  1.1
+
+    The field names to be merged are specified by columns
+    kv_delim is to delimit the key and value and '=' is default
+    item_delim is to delimit the Key-Value pairs, ',' is default
+    The new column name is specified by kv_col, 'kv_col' is default
+
+    >>> df.mf.collect_kv(columns=['k1', 'k2', 'k3', 'k5', 'k7', 'k9']).execute()
+       name   kv_col
+    0  name1  k1=1.0,k2=3.0,k5=10.0
+    1  name2  k2=3.0,k3=5.1
+    2  name3  k1=7.1,k7=8.2
+    3  name4  k2=1.2,k3=1.5
+    4  name5  k2=1.0,k9=1.1
+    """
+    columns_list = make_column_list(columns, data.dtypes) or []
+    non_exist_key = next(
+        (c for c in columns_list if c not in data.dtypes.index), no_default
+    )
+    if columns_list and non_exist_key is not no_default:
+        raise ValueError(f"Column {non_exist_key} specified is not a valid column.")
+    op = DataFrameCollectKv(
+        columns=columns,
+        kv_delim=kv_delim,
+        item_delim=item_delim,
+        kv_col=kv_col,
+    )
+    return op(data)

maxframe/dataframe/extensions/extract_kv.py

@@ -0,0 +1,177 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+
+import numpy as np
+import pandas as pd
+
+from ... import opcodes
+from ...core import EntityData, OutputType
+from ...serialization.serializables import AnyField, KeyField, StringField
+from ...utils import make_dtype, no_default
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+from ..utils import make_column_list
+
+
+class DataFrameExtractKv(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.EXTRACT_KV
+
+    columns = AnyField("columns", default=None)
+    kv_delim = StringField("kv_delim", default="=")
+    item_delim = StringField("item_delim", default=",")
+    dtype = AnyField("dtype", default=None)
+    fill_value = AnyField("fill_value", default=None)
+    errors = StringField("errors", default="raise")
+    # intermediate agg data
+    agg_results = KeyField("agg_results", default=None)
+
+    def __init__(self, kv_delim="=", item_delim=",", **kw):
+        super().__init__(kv_delim=kv_delim, item_delim=item_delim, **kw)
+        self.output_types = [OutputType.dataframe]
+
+    @classmethod
+    def _set_inputs(cls, op: "DataFrameExtractKv", inputs: List[EntityData]):
+        super()._set_inputs(op, inputs)
+        if op.agg_results is not None:
+            op.agg_results = inputs[-1]
+
+    def __call__(self, df):
+        shape = (df.shape[0], np.nan)
+        errors_arg = self.errors
+
+        def get_keys(row, cols, kv_delim, item_delim):
+            for col in cols:
+                if row[col] is not None:
+                    pairs = row[col].split(item_delim)
+                else:
+                    pairs = []
+                for pair in pairs:
+                    result = pair.split(kv_delim, 1)
+                    if len(result) == 2:
+                        yield f"{col}_{result[0]}"
+                    elif errors_arg == "raise":
+                        raise ValueError(f"Malformed data {pair} in column '{col}'.")
+
+        all_keys = df.mf.flatmap(
+            get_keys,
+            dtypes=pd.Series([str], index=["keys_cols"]),
+            cols=self.columns,
+            kv_delim=self.kv_delim,
+            item_delim=self.item_delim,
+        )
+        self.agg_results = all_keys.drop_duplicates().sort_values(by="keys_cols")
+        inputs = [df]
+        inputs.append(self.agg_results)
+        return self.new_dataframe(
+            inputs,
+            shape=shape,
+            dtypes=None,
+            index_value=df.index_value,
+            columns_value=None,
+        )
+
+
+def extract_kv(
+    data,
+    columns=None,
+    kv_delim="=",
+    item_delim=",",
+    dtype="float",
+    fill_value=None,
+    errors="raise",
+):
+    """
+    Extract values in key-value represented columns into standalone columns.
+    New column names will be the name of the key-value column followed by
+    an underscore and the key.
+
+    Parameters
+    ----------
+    columns : list, default None
+        The key-value columns to be extracted.
+    kv_delim : str, default '='
+        Delimiter between key and value.
+    item_delim : str, default ','
+        Delimiter between key-value pairs.
+    dtype : str
+        Type of value columns to generate.
+    fill_value : object, default None
+        Default value for missing key-value pairs.
+    errors : {'ignore', 'raise'}, default 'raise'
+        * If 'raise', then invalid parsing will raise an exception.
+        * If 'ignore', then invalid parsing will return the input.
+
+    Returns
+    -------
+    DataFrame
+        extracted data frame
+
+    See Also
+    --------
+    DataFrame.mf.collect_kv
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> import maxframe.dataframe as md
+
+    >>> df = md.DataFrame({"name": ["name1", "name2", "name3", "name4", "name5"],
+    ...                    "kv": ["k1=1.0,k2=3.0,k5=10.0",
+    ...                           "k2=3.0,k3=5.1",
+    ...                           "k1=7.1,k7=8.2",
+    ...                           "k2=1.2,k3=1.5",
+    ...                           "k2=1.0,k9=1.1"]})
+    >>> df.execute()
+       name   kv
+    0  name1  k1=1.0,k2=3.0,k5=10.0
+    1  name2  k2=3.0,k3=5.1
+    2  name3  k1=7.1,k7=8.2
+    3  name4  k2=1.2,k3=1.5
+    4  name5  k2=1.0,k9=1.1
+
+    The field names to be expanded are specified by columns
+    kv_delim is to delimit the key and value and '=' is default
+    item_delim is to delimit the Key-Value pairs, ',' is default
+    The output field name is the original field name connect with the key by "_"
+    fill_value is used to fill missing values, None is default
+
+    >>> df.mf.extract_kv(columns=['kv'], kv_delim='=', item_delim=',').execute()
+       name   kv_k1   kv_k2   kv_k3   kv_k5   kv_k7   kv_k9
+    0  name1  1.0     3.0     NaN     10.0    NaN     NaN
+    1  name2  NaN     3.0     5.1     NaN     NaN     NaN
+    2  name3  7.1     NaN     NaN     NaN     8.2     NaN
+    3  name4  NaN     1.2     1.5     NaN     NaN     NaN
+    4  name5  NaN     1.0     NaN     NaN     NaN     1.1
+    """
+    if columns is None:
+        columns = data.dtypes.index.tolist()
+    columns_list = make_column_list(columns, data.dtypes)
+    non_exist_key = next(
+        (c for c in columns_list if c not in data.dtypes.index), no_default
+    )
+    if non_exist_key is not no_default:
+        raise ValueError(f"Column {non_exist_key} specified is not a valid column.")
+    for col in columns_list:
+        if str(data.dtypes[col]) not in ("object", "string"):
+            raise ValueError(f"Column '{col}' must be of string type.")
+    op = DataFrameExtractKv(
+        columns=columns,
+        kv_delim=kv_delim,
+        item_delim=item_delim,
+        dtype=make_dtype(dtype),
+        fill_value=fill_value,
+        errors=errors,
+    )
+    return op(data)