maxframe 2.2.0__cp312-cp312-macosx_10_9_universal2.whl

maxframe/dataframe/extensions/cartesian_chunk.py

@@ -0,0 +1,153 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+
+import numpy as np
+import pandas as pd
+
+from ... import opcodes
+from ...core import EntityData, OutputType
+from ...serialization.serializables import (
+    DictField,
+    FunctionField,
+    KeyField,
+    TupleField,
+)
+from ...udf import BuiltinFunction
+from ...utils import quiet_stdio
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+from ..utils import (
+    build_df,
+    build_empty_df,
+    build_series,
+    parse_index,
+    validate_output_types,
+)
+
+
+class DataFrameCartesianChunk(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.CARTESIAN_CHUNK
+
+    left = KeyField("left")
+    right = KeyField("right")
+    func = FunctionField("func")
+    args = TupleField("args")
+    kwargs = DictField("kwargs")
+
+    def __init__(self, output_types=None, **kw):
+        super().__init__(_output_types=output_types, **kw)
+        if self.memory_scale is None:
+            self.memory_scale = 2.0
+
+    @classmethod
+    def _set_inputs(cls, op: "DataFrameCartesianChunk", inputs: List[EntityData]):
+        super()._set_inputs(op, inputs)
+        op.left, op.right = op.inputs[:2]
+
+    @staticmethod
+    def _build_test_obj(obj):
+        return (
+            build_df(obj, size=2)
+            if obj.ndim == 2
+            else build_series(obj, size=2, name=obj.name)
+        )
+
+    def has_custom_code(self) -> bool:
+        return not isinstance(self.func, BuiltinFunction)
+
+    def __call__(self, left, right, index=None, dtypes=None):
+        test_left = self._build_test_obj(left)
+        test_right = self._build_test_obj(right)
+        output_type = self._output_types[0] if self._output_types else None
+
+        if output_type == OutputType.df_or_series:
+            return self.new_df_or_series([left, right])
+
+        # try run to infer meta
+        try:
+            with np.errstate(all="ignore"), quiet_stdio():
+                obj = self.func(test_left, test_right, *self.args, **self.kwargs)
+        except:  # noqa: E722  # nosec  # pylint: disable=bare-except
+            if output_type == OutputType.series:
+                obj = pd.Series([], dtype=np.dtype(object))
+            elif output_type == OutputType.dataframe and dtypes is not None:
+                obj = build_empty_df(dtypes)
+            else:
+                raise TypeError(
+                    "Cannot determine `output_type`, "
+                    "you have to specify it as `dataframe` or `series`, "
+                    "for dataframe, `dtypes` is required as well "
+                    "if output_type='dataframe'"
+                )
+
+        if getattr(obj, "ndim", 0) == 1 or output_type == OutputType.series:
+            shape = self.kwargs.pop("shape", (np.nan,))
+            if index is None:
+                index = obj.index
+            index_value = parse_index(
+                index, left, right, self.func, self.args, self.kwargs
+            )
+            return self.new_series(
+                [left, right],
+                dtype=obj.dtype,
+                shape=shape,
+                index_value=index_value,
+                name=obj.name,
+            )
+        else:
+            dtypes = dtypes if dtypes is not None else obj.dtypes
+            # dataframe
+            shape = (np.nan, len(dtypes))
+            columns_value = parse_index(dtypes.index, store_data=True)
+            if index is None:
+                index = obj.index
+            index_value = parse_index(
+                index, left, right, self.func, self.args, self.kwargs
+            )
+            return self.new_dataframe(
+                [left, right],
+                shape=shape,
+                dtypes=dtypes,
+                index_value=index_value,
+                columns_value=columns_value,
+            )
+
+
+def cartesian_chunk(left, right, func, skip_infer=False, args=(), **kwargs):
+    output_type = kwargs.pop("output_type", None)
+    output_types = kwargs.pop("output_types", None)
+    object_type = kwargs.pop("object_type", None)
+    output_types = validate_output_types(
+        output_type=output_type, output_types=output_types, object_type=object_type
+    )
+    output_type = output_types[0] if output_types else None
+    if output_type:
+        output_types = [output_type]
+    elif skip_infer:
+        output_types = [OutputType.df_or_series]
+    index = kwargs.pop("index", None)
+    dtypes = kwargs.pop("dtypes", None)
+    memory_scale = kwargs.pop("memory_scale", None)
+
+    op = DataFrameCartesianChunk(
+        left=left,
+        right=right,
+        func=func,
+        args=args,
+        kwargs=kwargs,
+        output_types=output_types,
+        memory_scale=memory_scale,
+    )
+    return op(left, right, index=index, dtypes=dtypes)

maxframe/dataframe/extensions/collect_kv.py

@@ -0,0 +1,126 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+import pandas as pd
+
+from ... import opcodes
+from ...serialization.serializables import AnyField, StringField
+from ...utils import no_default
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+from ..utils import make_column_list
+
+
+class DataFrameCollectKv(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.COLLECT_KV
+
+    columns = AnyField("columns", default=None)
+    kv_delim = StringField("kv_delim", default=None)
+    item_delim = StringField("item_delim", default=None)
+    kv_col = StringField("kv_col", default=None)
+
+    def __call__(self, df):
+        if self.columns is None:
+            cols = list(df.dtypes.index)
+        else:
+            cols = self.columns if isinstance(self.columns, list) else [self.columns]
+        new_dtypes = df.dtypes.drop(cols, errors="ignore")
+        new_dtypes = pd.concat(
+            [new_dtypes, pd.Series([np.dtype("object")], index=[self.kv_col])]
+        )
+        shape = (df.shape[0], len(new_dtypes))
+        return self.new_dataframe(
+            [df],
+            shape=shape,
+            dtypes=new_dtypes,
+            index_value=df.index_value,
+            columns_value=new_dtypes.index,
+        )
+
+
+def collect_kv(
+    data,
+    columns=None,
+    kv_delim="=",
+    item_delim=",",
+    kv_col="kv_col",
+):
+    """
+    Merge values in specified columns into a key-value represented column.
+
+    Parameters
+    ----------
+    columns : list, default None
+        The columns to be merged.
+    kv_delim : str, default '='
+        Delimiter between key and value.
+    item_delim : str, default ','
+        Delimiter between key-value pairs.
+    kv_col : str, default 'kv_col'
+        Name of the new key-value column
+
+    Returns
+    -------
+    DataFrame
+        converted data frame
+
+    See Also
+    --------
+    DataFrame.mf.extract_kv
+
+    Examples
+    -------
+    >>> import maxframe.dataframe as md
+
+    >>> df = md.DataFrame({"name": ["name1", "name2", "name3", "name4", "name5"],
+    ...                    "k1": [1.0, NaN, 7.1, NaN, NaN],
+    ...                    "k2": [3.0, 3.0, NaN, 1.2, 1.0],
+    ...                    "k3": [NaN, 5.1, NaN, 1.5, NaN],
+    ...                    "k5": [10.0, NaN, NaN, NaN, NaN,],
+    ...                    "k7": [NaN, NaN, 8.2, NaN, NaN, ],
+    ...                    "k9": [NaN, NaN, NaN, NaN, 1.1]})
+    >>> df.execute()
+       name   k1   k2   k3   k5    k7   k9
+    0  name1  1.0  3.0  NaN  10.0  NaN  NaN
+    1  name2  NaN  3.0  5.1  NaN   NaN  NaN
+    2  name3  7.1  NaN  NaN  NaN   8.2  NaN
+    3  name4  NaN  1.2  1.5  NaN   NaN  NaN
+    4  name5  NaN  1.0  NaN  NaN   NaN  1.1
+
+    The field names to be merged are specified by columns
+    kv_delim is to delimit the key and value and '=' is default
+    item_delim is to delimit the Key-Value pairs, ',' is default
+    The new column name is specified by kv_col, 'kv_col' is default
+
+    >>> df.mf.collect_kv(columns=['k1', 'k2', 'k3', 'k5', 'k7', 'k9']).execute()
+       name   kv_col
+    0  name1  k1=1.0,k2=3.0,k5=10.0
+    1  name2  k2=3.0,k3=5.1
+    2  name3  k1=7.1,k7=8.2
+    3  name4  k2=1.2,k3=1.5
+    4  name5  k2=1.0,k9=1.1
+    """
+    columns_list = make_column_list(columns, data.dtypes) or []
+    non_exist_key = next(
+        (c for c in columns_list if c not in data.dtypes.index), no_default
+    )
+    if columns_list and non_exist_key is not no_default:
+        raise ValueError(f"Column {non_exist_key} specified is not a valid column.")
+    op = DataFrameCollectKv(
+        columns=columns,
+        kv_delim=kv_delim,
+        item_delim=item_delim,
+        kv_col=kv_col,
+    )
+    return op(data)

maxframe/dataframe/extensions/extract_kv.py

@@ -0,0 +1,177 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+
+import numpy as np
+import pandas as pd
+
+from ... import opcodes
+from ...core import EntityData, OutputType
+from ...serialization.serializables import AnyField, KeyField, StringField
+from ...utils import make_dtype, no_default
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+from ..utils import make_column_list
+
+
+class DataFrameExtractKv(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.EXTRACT_KV
+
+    columns = AnyField("columns", default=None)
+    kv_delim = StringField("kv_delim", default="=")
+    item_delim = StringField("item_delim", default=",")
+    dtype = AnyField("dtype", default=None)
+    fill_value = AnyField("fill_value", default=None)
+    errors = StringField("errors", default="raise")
+    # intermediate agg data
+    agg_results = KeyField("agg_results", default=None)
+
+    def __init__(self, kv_delim="=", item_delim=",", **kw):
+        super().__init__(kv_delim=kv_delim, item_delim=item_delim, **kw)
+        self.output_types = [OutputType.dataframe]
+
+    @classmethod
+    def _set_inputs(cls, op: "DataFrameExtractKv", inputs: List[EntityData]):
+        super()._set_inputs(op, inputs)
+        if op.agg_results is not None:
+            op.agg_results = inputs[-1]
+
+    def __call__(self, df):
+        shape = (df.shape[0], np.nan)
+        errors_arg = self.errors
+
+        def get_keys(row, cols, kv_delim, item_delim):
+            for col in cols:
+                if row[col] is not None:
+                    pairs = row[col].split(item_delim)
+                else:
+                    pairs = []
+                for pair in pairs:
+                    result = pair.split(kv_delim, 1)
+                    if len(result) == 2:
+                        yield f"{col}_{result[0]}"
+                    elif errors_arg == "raise":
+                        raise ValueError(f"Malformed data {pair} in column '{col}'.")
+
+        all_keys = df.mf.flatmap(
+            get_keys,
+            dtypes=pd.Series([str], index=["keys_cols"]),
+            cols=self.columns,
+            kv_delim=self.kv_delim,
+            item_delim=self.item_delim,
+        )
+        self.agg_results = all_keys.drop_duplicates().sort_values(by="keys_cols")
+        inputs = [df]
+        inputs.append(self.agg_results)
+        return self.new_dataframe(
+            inputs,
+            shape=shape,
+            dtypes=None,
+            index_value=df.index_value,
+            columns_value=None,
+        )
+
+
+def extract_kv(
+    data,
+    columns=None,
+    kv_delim="=",
+    item_delim=",",
+    dtype="float",
+    fill_value=None,
+    errors="raise",
+):
+    """
+    Extract values in key-value represented columns into standalone columns.
+    New column names will be the name of the key-value column followed by
+    an underscore and the key.
+
+    Parameters
+    ----------
+    columns : list, default None
+        The key-value columns to be extracted.
+    kv_delim : str, default '='
+        Delimiter between key and value.
+    item_delim : str, default ','
+        Delimiter between key-value pairs.
+    dtype : str
+        Type of value columns to generate.
+    fill_value : object, default None
+        Default value for missing key-value pairs.
+    errors : {'ignore', 'raise'}, default 'raise'
+        * If 'raise', then invalid parsing will raise an exception.
+        * If 'ignore', then invalid parsing will return the input.
+
+    Returns
+    -------
+    DataFrame
+        extracted data frame
+
+    See Also
+    --------
+    DataFrame.mf.collect_kv
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> import maxframe.dataframe as md
+
+    >>> df = md.DataFrame({"name": ["name1", "name2", "name3", "name4", "name5"],
+    ...                    "kv": ["k1=1.0,k2=3.0,k5=10.0",
+    ...                           "k2=3.0,k3=5.1",
+    ...                           "k1=7.1,k7=8.2",
+    ...                           "k2=1.2,k3=1.5",
+    ...                           "k2=1.0,k9=1.1"]})
+    >>> df.execute()
+       name   kv
+    0  name1  k1=1.0,k2=3.0,k5=10.0
+    1  name2  k2=3.0,k3=5.1
+    2  name3  k1=7.1,k7=8.2
+    3  name4  k2=1.2,k3=1.5
+    4  name5  k2=1.0,k9=1.1
+
+    The field names to be expanded are specified by columns
+    kv_delim is to delimit the key and value and '=' is default
+    item_delim is to delimit the Key-Value pairs, ',' is default
+    The output field name is the original field name connect with the key by "_"
+    fill_value is used to fill missing values, None is default
+
+    >>> df.mf.extract_kv(columns=['kv'], kv_delim='=', item_delim=',').execute()
+       name   kv_k1   kv_k2   kv_k3   kv_k5   kv_k7   kv_k9
+    0  name1  1.0     3.0     NaN     10.0    NaN     NaN
+    1  name2  NaN     3.0     5.1     NaN     NaN     NaN
+    2  name3  7.1     NaN     NaN     NaN     8.2     NaN
+    3  name4  NaN     1.2     1.5     NaN     NaN     NaN
+    4  name5  NaN     1.0     NaN     NaN     NaN     1.1
+    """
+    if columns is None:
+        columns = data.dtypes.index.tolist()
+    columns_list = make_column_list(columns, data.dtypes)
+    non_exist_key = next(
+        (c for c in columns_list if c not in data.dtypes.index), no_default
+    )
+    if non_exist_key is not no_default:
+        raise ValueError(f"Column {non_exist_key} specified is not a valid column.")
+    for col in columns_list:
+        if str(data.dtypes[col]) not in ("object", "string"):
+            raise ValueError(f"Column '{col}' must be of string type.")
+    op = DataFrameExtractKv(
+        columns=columns,
+        kv_delim=kv_delim,
+        item_delim=item_delim,
+        dtype=make_dtype(dtype),
+        fill_value=fill_value,
+        errors=errors,
+    )
+    return op(data)

maxframe/dataframe/extensions/flatjson.py

@@ -0,0 +1,132 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+
+from ... import opcodes
+from ...core import OutputType
+from ...serialization.serializables import ListField
+from ...serialization.serializables.field_type import FieldTypes
+from ...utils import make_dtype, make_dtypes
+from ..core import DataFrame
+from ..operators import DataFrameOperator, DataFrameOperatorMixin
+from ..utils import parse_index
+
+
+class SeriesFlatJSONOperator(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.FLATJSON
+
+    query_paths = ListField("query_paths", field_type=FieldTypes.string, default=None)
+
+    def __call__(self, series, dtypes):
+        if self._output_types[0] == OutputType.series:
+            name, dtype = dtypes
+            return self.new_series(
+                [series],
+                shape=series.shape,
+                index_value=series.index_value,
+                name=name,
+                dtype=make_dtype(dtype),
+            )
+        return self.new_dataframe(
+            [series],
+            shape=(series.shape[0], len(dtypes)),
+            index_value=series.index_value,
+            columns_value=parse_index(dtypes.index, store_data=True),
+            dtypes=make_dtypes(dtypes),
+        )
+
+
+def series_flatjson(
+    series,
+    query_paths: List[str],
+    dtypes=None,
+    dtype=None,
+    name: str = None,
+) -> DataFrame:
+    """
+    Flat JSON object in the series to a dataframe according to JSON query.
+
+    Parameters
+    ----------
+    series : Series
+        The series of json strings.
+
+    query_paths: List[str] or str
+        The JSON query paths for each generated column. The path format should follow
+        [RFC9535](https://datatracker.ietf.org/doc/rfc9535/).
+
+    dtypes : Series, default None
+        Specify dtypes of returned DataFrame. Can't work with dtype.
+
+    dtype : numpy.dtype, default None
+        Specify dtype of returned Series. Can't work with dtypes.
+
+    name : str, default None
+        Specify name of the returned Series.
+
+    Returns
+    -------
+    DataFrame or Series
+        Result of DataFrame when dtypes specified, else Series.
+
+    Examples
+    --------
+    >>> import maxframe.dataframe as md
+    >>> import pandas as pd
+    >>> s = md.Series(
+    ...     [
+    ...         '{"age": 24, "gender": "male", "graduated": false}',
+    ...         '{"age": 25, "gender": "female", "graduated": true}',
+    ...     ]
+    ... )
+    >>> s.execute()
+    0    {"age": 24, "gender": "male", "graduated": false}
+    1    {"age": 25, "gender": "female", "graduated": true}
+    dtype: object
+
+    >>> df = s.mf.flatjson(
+    ...    ["$.age", "$.gender", "$.graduated"],
+    ...    dtypes=pd.Series(["int32", "object", "bool"], index=["age", "gender", "graduated"]),
+    ... )
+    >>> df.execute()
+        age  gender  graduated
+    0   24    male       True
+    1   25  female       True
+
+    >>> s2 = s.mf.flatjson("$.age", name="age", dtype="int32")
+    >>> s2.execute()
+    0    24
+    1    25
+    Name: age, dtype: int32
+    """
+    if isinstance(query_paths, str):
+        query_paths = [query_paths]
+    if dtypes is not None and dtype is not None:
+        raise ValueError("Both dtypes and dtype cannot be specified at the same time.")
+    if dtype is not None:
+        if len(query_paths) != 1:
+            raise ValueError("query_paths should have only one path if dtype is set")
+        output_type = OutputType.series
+    elif dtypes is not None:
+        if len(dtypes) != len(query_paths):
+            raise ValueError("query_paths and dtypes should have same length")
+        output_type = OutputType.dataframe
+    else:
+        raise ValueError("dtypes or dtype should be specified")
+
+    dtypes = (name, dtype) if dtype is not None else dtypes
+    return SeriesFlatJSONOperator(query_paths=query_paths, _output_types=[output_type])(
+        series, dtypes
+    )