maxframe 2.4.0rc1__cp312-cp312-win32.whl

maxframe/learn/contrib/__init__.py

@@ -0,0 +1,17 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from . import graph, lightgbm, llm, pytorch
+
+del graph, lightgbm, llm, pytorch

maxframe/learn/contrib/graph/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .connected_components import connected_components

maxframe/learn/contrib/graph/connected_components.py

@@ -0,0 +1,216 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+import pandas as pd
+
+from maxframe import opcodes
+
+from ....core import OutputType
+from ....dataframe.operators import DataFrameOperator, DataFrameOperatorMixin
+from ....dataframe.utils import parse_index
+from ....serialization.serializables import Int32Field, StringField
+from ....utils import make_dtypes
+
+
+class DataFrameConnectedComponentsOperator(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.CONNECTED_COMPONENTS
+
+    vertex_col1 = StringField("vertex_col1", default=None)
+    vertex_col2 = StringField("vertex_col2", default=None)
+    max_iter = Int32Field("max_iter", default=6)
+
+    def __call__(self, df):
+        node_id_dtype = df.dtypes[self.vertex_col1]
+        dtypes = make_dtypes({"id": node_id_dtype, "component": node_id_dtype})
+        # this will return a dataframe and a bool flag
+        new_dataframe_tileable_kw = {
+            "shape": (np.nan, 2),
+            "index_value": parse_index(pd.RangeIndex(0)),
+            "columns_value": parse_index(dtypes.index, store_data=True),
+            "dtypes": dtypes,
+        }
+        new_scalar_tileable_kw = {"dtype": np.dtype(np.bool_), "shape": ()}
+        return self.new_tileables(
+            [df],
+            kws=[new_dataframe_tileable_kw, new_scalar_tileable_kw],
+        )
+
+    @property
+    def output_limit(self):
+        return 2
+
+
+def connected_components(
+    dataframe, vertex_col1: str, vertex_col2: str, max_iter: int = 6
+):
+    """
+    The connected components algorithm labels each node as belonging to a specific connected component with the ID of
+    its lowest-numbered vertex.
+
+    Parameters
+    ----------
+    dataframe : DataFrame
+        A DataFrame containing the edges of the graph.
+
+    vertex_col1 : str
+        The name of the column in `dataframe` that contains the one of edge vertices. The column value must be an
+        integer.
+
+    vertex_col2 : str
+        The name of the column in `dataframe` that contains the other one of edge vertices. The column value must be an
+        integer.
+
+    max_iter : int
+        The algorithm use large and small star transformation to find all connected components, `max_iter`
+        controls the max round of the iterations before finds all edges. Default is 6.
+
+
+    Returns
+    -------
+    DataFrame
+        Return dataFrame contains all connected component edges by two columns `id` and `component`. `component` is
+        the lowest-numbered vertex in the connected components.
+
+    Notes
+    -------
+    After `execute()`, the dataframe has a bool member `flag` to indicate if the `connected_components` already
+    converged in `max_iter` rounds. `True` means the dataframe already contains all edges of the connected components.
+    If `False` you can run `connected_components` more times to reach the converged state.
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> import maxframe.dataframe as md
+    >>> import maxframe.learn.contrib.graph.connected_components
+    >>> df = md.DataFrame({'x': [4, 1], 'y': [0, 4]})
+    >>> df.execute()
+       x  y
+    0  4  1
+    1  0  4
+
+    Get connected components with 1 round iteration.
+
+    >>> components, converged = connected_components(df, "x", "y", 1)
+    >>> session.execute(components, converged)
+    >>> components
+       A   B
+    0  1   0
+    1  4   0
+
+    >>> converged
+    True
+
+    Sometimes, a single iteration may not be sufficient to propagate the connectivity of all edges.
+    By default, `connected_components` performs 6 iterations of calculations.
+    If you are unsure whether the connected components have converged, you can check the `flag` variable in
+    the output DataFrame after calling `execute()`.
+
+    >>> df = md.DataFrame({'x': [4, 1, 7, 5, 8, 11, 11], 'y': [0, 4, 4, 7, 7, 9, 13]})
+    >>> df.execute()
+        x   y
+    0   4   0
+    1   1   4
+    2   7   4
+    3   5   7
+    4   8   7
+    5  11   9
+    6  11  13
+
+    >>> components, converged = connected_components(df, "x", "y", 1)
+    >>> session.execute(components, converged)
+    >>> components
+       id  component
+    0   4          0
+    1   7          0
+    2   8          4
+    3  13          9
+    4   1          0
+    5   5          0
+    6  11          9
+
+    If `flag` is True, it means convergence has been achieved.
+
+    >>> converged
+    False
+
+    You can determine whether to continue iterating or to use a larger number of iterations
+    (but not too large, which would result in wasted computational overhead).
+
+    >>> components, converged = connected_components(components, "id", "component", 1)
+    >>> session.execute(components, converged)
+    >>> components
+       id  component
+    0   4          0
+    1   7          0
+    2  13          9
+    3   1          0
+    4   5          0
+    5  11          9
+    6   8          0
+
+    >>> components, converged = connected_components(df, "x", "y")
+    >>> session.execute(components, converged)
+    >>> components
+       id  component
+    0   4          0
+    1   7          0
+    2  13          9
+    3   1          0
+    4   5          0
+    5  11          9
+    6   8          0
+    """
+
+    # Check if vertex columns are provided
+    if not vertex_col1 or not vertex_col2:
+        raise ValueError("Both vertex_col1 and vertex_col2 must be provided.")
+
+    # Check if max_iter is provided and within the valid range
+    if max_iter is None:
+        raise ValueError("max_iter must be provided.")
+    if not (1 <= max_iter <= 50):
+        raise ValueError("max_iter must be an integer between 1 and 50.")
+
+    # Verify that the vertex columns exist in the dataframe
+    missing_cols = [
+        col for col in (vertex_col1, vertex_col2) if col not in dataframe.dtypes
+    ]
+    if missing_cols:
+        raise ValueError(
+            f"The following required columns {missing_cols} are not in {list(dataframe.dtypes.index)}"
+        )
+
+    # Ensure that the vertex columns are of integer type
+    # TODO support string dtype
+    incorrect_dtypes = [
+        col
+        for col in (vertex_col1, vertex_col2)
+        if dataframe[col].dtype != np.dtype("int")
+    ]
+    if incorrect_dtypes:
+        dtypes_str = ", ".join(str(dataframe[col].dtype) for col in incorrect_dtypes)
+        raise ValueError(
+            f"Columns {incorrect_dtypes} should be of integer type, but found {dtypes_str}."
+        )
+
+    op = DataFrameConnectedComponentsOperator(
+        vertex_col1=vertex_col1,
+        vertex_col2=vertex_col2,
+        _output_types=[OutputType.dataframe, OutputType.scalar],
+        max_iter=max_iter,
+    )
+    return op(
+        dataframe,
+    )

maxframe/learn/contrib/graph/tests/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

maxframe/learn/contrib/graph/tests/test_connected_components.py

@@ -0,0 +1,53 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+import pytest
+
+from ..... import dataframe as md
+from .....dataframe.core import DataFrameData
+from .....tensor.core import TensorData
+from .. import connected_components
+
+
+@pytest.fixture
+def df1():
+    return md.DataFrame({"a": [1, 2, 3], "b": [1, 2, 3], "c": [1, 2, 3]})
+
+
+@pytest.fixture
+def df2():
+    return md.DataFrame(
+        [[1, "2"], [1, "2"]],
+        columns=["a", "b"],
+    )
+
+
+def test_connected_components(df1, df2):
+    edges, flag = connected_components(df1, "a", "b")
+    assert edges.op.max_iter == 6
+    assert edges.shape == (np.nan, 2)
+    assert isinstance(edges.data, DataFrameData)
+    assert isinstance(flag.data, TensorData)
+    assert flag.shape == ()
+    assert "id" in edges.dtypes and "component" in edges.dtypes
+
+    with pytest.raises(ValueError):
+        connected_components(df1, "a", "x")
+
+    with pytest.raises(ValueError):
+        connected_components(df1, "a", "b", 0)
+
+    with pytest.raises(ValueError):
+        connected_components(df2, "a", "b")

maxframe/learn/contrib/lightgbm/__init__.py

@@ -0,0 +1,33 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ._predict import predict
+from ._train import train
+from .dataset import Dataset
+
+# isort: off
+from ..utils import config_mod_getattr as _config_mod_getattr
+
+_config_mod_getattr(
+    {
+        "LGBMClassifier": ".classifier.LGBMClassifier",
+        "LGBMRegressor": ".regressor.LGBMRegressor",
+        # "LGBMRanker": ".ranker.LGBMRanker",
+    },
+    globals(),
+)
+
+del _config_mod_getattr
+
+__all__ = ["Dataset", "LGBMClassifier", "LGBMRegressor", "predict", "train"]

maxframe/learn/contrib/lightgbm/_predict.py

@@ -0,0 +1,138 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Dict, List
+
+import numpy as np
+
+from .... import opcodes
+from ....core import OutputType
+from ....core.operator import Operator, TileableOperatorMixin
+from ....serialization.serializables import BoolField, DictField, Int32Field, KeyField
+from ....tensor.core import TensorOrder
+from ....typing_ import EntityType
+from ..models import to_remote_model
+from .dataset import check_data
+
+
+class LGBMPredict(Operator, TileableOperatorMixin):
+    _op_type_ = opcodes.LGBM_PREDICT
+    _op_module_ = "learn.contrib.lightgbm"
+    output_dtype = np.dtype(np.float64)
+
+    data = KeyField("data", default=None)
+    booster = KeyField("booster", default=None)
+    start_iteration = Int32Field("start_iteration", default=0)
+    num_iterations = Int32Field("num_iterations", default=None)
+    raw_score = BoolField("raw_score", default=False)
+    pred_leaf = BoolField("pred_leaf", default=False)
+    pred_contrib = BoolField("pred_contrib", default=False)
+    validate_features = BoolField("validate_features", default=False)
+    kwds = DictField("kwds", default=None)
+
+    def __init__(self, output_types=None, **kw):
+        super().__init__(_output_types=output_types, **kw)
+
+    def has_custom_code(self) -> bool:
+        return True
+
+    @property
+    def output_limit(self) -> int:
+        return 1 + self.pred_leaf + self.pred_contrib
+
+    @classmethod
+    def _set_inputs(cls, op: "LGBMPredict", inputs: List[EntityType]):
+        super()._set_inputs(op, inputs)
+        it = iter(inputs)
+        op.data = next(it)
+        op.booster = next(it)
+
+    def __call__(self):
+        num_class = getattr(self.booster.op, "num_class", None)
+
+        result_kw: Dict[str, Any] = {
+            "dtype": self.output_dtype,
+            "order": TensorOrder.C_ORDER,
+        }
+        kws = [result_kw]
+
+        if num_class is not None:
+            num_class = int(num_class)
+        if num_class is not None and num_class > 2:
+            result_kw["shape"] = (self.data.shape[0], num_class)
+        else:
+            result_kw["shape"] = (self.data.shape[0],)
+
+        if self.pred_leaf:
+            kws.append(
+                {
+                    "shape": (np.nan, np.nan),
+                    "dtype": np.dtype(np.float_),
+                    "order": TensorOrder.C_ORDER,
+                }
+            )
+        if self.pred_contrib:
+            kws.append(
+                {
+                    "shape": (np.nan, np.nan),
+                    "dtype": np.dtype(np.float_),
+                    "order": TensorOrder.C_ORDER,
+                }
+            )
+
+        return self.new_tileables([self.data, self.booster], kws=kws)
+
+
+def predict(
+    booster,
+    data,
+    raw_score: bool = False,
+    start_iteration: int = 0,
+    num_iteration: int = None,
+    pred_leaf: bool = False,
+    pred_contrib: bool = False,
+    validate_features: bool = False,
+    **kwargs,
+):
+    import lightgbm
+
+    from .core import Booster, BoosterData
+
+    if not isinstance(booster, (Booster, BoosterData, lightgbm.Booster)):
+        raise TypeError(
+            f"model has to be a lightgbm.Booster, got {type(booster)} instead"
+        )
+    elif isinstance(booster, lightgbm.Booster):
+        booster = to_remote_model(booster, model_cls=Booster)
+    proba = kwargs.pop("proba", False)
+
+    data = check_data(data)
+
+    op = LGBMPredict(
+        data=data,
+        booster=booster,
+        output_types=[OutputType.tensor],
+        proba=proba,
+        raw_score=raw_score,
+        start_iteration=start_iteration,
+        num_iteration=num_iteration,
+        pred_leaf=pred_leaf,
+        pred_contrib=pred_contrib,
+        validate_features=validate_features,
+        kwds=kwargs,
+    )
+    results = op()
+    if len(results) == 1:
+        return results[0]
+    return results

maxframe/learn/contrib/lightgbm/_train.py

@@ -0,0 +1,163 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import itertools
+import logging
+from typing import List
+
+from .... import opcodes
+from ....core import OutputType
+from ....core.operator import ObjectOperator, ObjectOperatorMixin
+from ....serialization.serializables import (
+    AnyField,
+    BoolField,
+    DictField,
+    FieldTypes,
+    FunctionField,
+    Int32Field,
+    KeyField,
+    ListField,
+    StringField,
+)
+from ....typing_ import EntityType
+from ..models import to_remote_model
+from ..utils import TrainingCallback
+from .core import Booster, BoosterData, LGBMScikitLearnBase
+
+logger = logging.getLogger(__name__)
+
+
+class LGBMTrain(ObjectOperator, ObjectOperatorMixin):
+    _op_type_ = opcodes.LGBM_TRAIN
+    _op_module_ = "learn.contrib.lightgbm"
+
+    params = DictField("params", key_type=FieldTypes.string, default_factory=dict)
+    train_set = KeyField("train_set", default=None)
+    num_boost_round = Int32Field("num_boost_round", default=None)
+    valid_sets = ListField("valid_sets", FieldTypes.key, default_factory=list)
+    valid_names = ListField("valid_names", FieldTypes.string, default_factory=list)
+    feval = FunctionField("feval", default=None)
+    init_model = KeyField("init_model", default=None)
+    feature_name = AnyField("feature_name", default=None)
+    categorical_feature = AnyField("categorical_feature", default=None)
+    keep_training_booster = BoolField("keep_training_booster", default=False)
+    callbacks = ListField(
+        "callbacks",
+        field_type=FunctionField.field_type,
+        default=None,
+        on_serialize=TrainingCallback.from_local,
+    )
+    tree_learner = StringField("tree_learner", default=None)
+    timeout = Int32Field("timeout", default=None)
+    # indicating shape of the predicted data of the model
+    num_class = Int32Field("num_class", default=None)
+
+    def __init__(self, gpu=None, **kw):
+        super().__init__(gpu=gpu, **kw)
+        if self.output_types is None:
+            self.output_types = [OutputType.object]
+        if self.has_evals_result:
+            self.output_types.append(OutputType.object)
+
+    @classmethod
+    def _set_inputs(cls, op: "LGBMTrain", inputs: List[EntityType]):
+        super()._set_inputs(op, inputs)
+        input_it = iter(op.inputs)
+        op.train_set = next(input_it)
+        op.valid_sets = list(itertools.islice(input_it, len(op.valid_sets or [])))
+        if op.init_model is not None:
+            op.init_model = next(input_it)
+
+    def __call__(self, evals_result):
+        inputs = [self.train_set]
+        if self.has_evals_result:
+            inputs.extend(self.valid_sets)
+        if self.init_model is not None:
+            inputs.append(self.init_model)
+        kws = [{"object_class": Booster}, {}]
+        return self.new_tileables(inputs, kws=kws, evals_result=evals_result)[0]
+
+    @property
+    def output_limit(self):
+        return 2 if self.has_evals_result else 1
+
+    def has_custom_code(self) -> bool:
+        if not self.callbacks:
+            return False
+        return any(
+            not isinstance(cb, TrainingCallback) or cb.has_custom_code()
+            for cb in self.callbacks
+        )
+
+    @property
+    def has_evals_result(self) -> bool:
+        return bool(self.valid_sets)
+
+
+def _get_lgbm_booster(init_model):
+    import lightgbm
+
+    if isinstance(init_model, (LGBMScikitLearnBase, lightgbm.LGBMModel)):
+        init_model = init_model.booster_
+
+    if isinstance(init_model, (Booster, BoosterData)):
+        return init_model
+    elif isinstance(init_model, lightgbm.Booster):
+        return to_remote_model(init_model, model_cls=Booster)
+    raise ValueError(f"Cannot use {type(init_model)} as init_model")
+
+
+def train(
+    params,
+    train_set,
+    num_boost_round=100,
+    valid_sets=None,
+    valid_names=None,
+    feval=None,
+    init_model=None,
+    keep_training_booster=False,
+    callbacks=None,
+    num_class=2,
+    evals_result=None,
+    **kwargs,
+):
+    evals_result = evals_result if evals_result is not None else dict()
+    timeout = kwargs.pop("timeout", 120)
+    session = kwargs.pop("session", None)
+    run_kwargs = kwargs.pop("run_kwargs", dict())
+    if valid_sets and valid_names:
+        if not isinstance(valid_names, list):
+            raise TypeError("valid_names must be a list of strings")
+        if len(valid_names) != len(valid_sets):
+            raise ValueError("size of valid_names must match size of valid_sets")
+    if init_model is not None:
+        init_model = _get_lgbm_booster(init_model)
+    data = LGBMTrain(
+        params=params,
+        train_set=train_set,
+        num_boost_round=num_boost_round,
+        valid_sets=valid_sets,
+        valid_names=valid_names,
+        feval=feval,
+        init_model=init_model,
+        keep_training_booster=keep_training_booster,
+        callbacks=callbacks,
+        num_class=num_class,
+        evals_result=evals_result,
+        timeout=timeout,
+        **kwargs,
+    )(evals_result)
+    if valid_sets:
+        data.execute(session=session, **run_kwargs)
+    return data