maxframe 1.3.1__cp39-cp39-macosx_10_9_universal2.whl → 2.0.0__cp39-cp39-macosx_10_9_universal2.whl

maxframe/io/odpsio/tests/test_schema.py

@@ -21,13 +21,15 @@ from odps import types as odps_types
 from .... import dataframe as md
 from .... import tensor as mt
 from ....core import OutputType
-from ....lib.dtypes_extension import dict_
+from ....lib.dtypes_extension import ArrowDtype, dict_, list_
 from ....utils import pd_release_version
 from ..schema import (
     arrow_schema_to_odps_schema,
     build_dataframe_table_meta,
     build_table_column_name,
     odps_schema_to_arrow_schema,
+    odps_schema_to_pandas_dtypes,
+    pandas_dtypes_to_arrow_schema,
     pandas_to_odps_schema,
     pandas_types_to_arrow_schema,
 )
@@ -42,6 +44,8 @@ def _wrap_maxframe_obj(obj, wrap="no"):
         obj = md.Series(obj)
     elif isinstance(obj, pd.Index):
         obj = md.Index(obj)
+    elif isinstance(obj, np.ndarray):
+        obj = mt.tensor(obj)
     else:
         obj = mt.scalar(obj)
     if wrap == "data":
@@ -191,6 +195,24 @@ def test_pandas_to_odps_schema_scalar(wrap_obj):
     assert meta.pd_index_level_names == [None]
 
 
+@pytest.mark.parametrize("wrap_obj", ["no", "yes", "data"])
+def test_pandas_to_odps_schema_tensor(wrap_obj):
+    data = np.array([1, 2, 3])
+
+    test_tensor = _wrap_maxframe_obj(data, wrap=wrap_obj)
+    if wrap_obj != "no":
+        test_tensor.op.data = None
+
+    schema, meta = pandas_to_odps_schema(test_tensor, unknown_as_string=True)
+    assert schema.columns[0].name == "_idx_0"
+    assert schema.columns[0].type.name == "bigint"
+    assert meta.type == OutputType.tensor
+    assert meta.table_column_names == []
+    assert meta.table_index_column_names == ["_idx_0"]
+    assert meta.pd_column_level_names == []
+    assert meta.pd_index_level_names == [None]
+
+
 def test_odps_arrow_schema_conversion():
     odps_schema = odps_types.OdpsSchema(
         [
@@ -275,6 +297,95 @@ def test_odps_arrow_schema_conversion():
         arrow_schema_to_odps_schema(pa.schema([("col1", pa.float16())]))
 
 
+def test_odps_pandas_schema_conversion():
+    odps_schema = odps_types.OdpsSchema(
+        [
+            odps_types.Column("col1", "string"),
+            odps_types.Column("col2", "binary"),
+            odps_types.Column("col3", "tinyint"),
+            odps_types.Column("col4", "smallint"),
+            odps_types.Column("col5", "int"),
+            odps_types.Column("col6", "bigint"),
+            odps_types.Column("col7", "boolean"),
+            odps_types.Column("col8", "float"),
+            odps_types.Column("col9", "double"),
+            # odps_types.Column("col10", "date"),
+            odps_types.Column("col11", "datetime"),
+            odps_types.Column("col12", "timestamp"),
+            # odps_types.Column("col13", "decimal(10, 2)"),
+            odps_types.Column("col14", "array<string>"),
+            odps_types.Column("col15", "map<string, bigint>"),
+            # odps_types.Column("col16", "struct<a1: string, a2: map<string, bigint>>"),
+            # odps_types.Column("col17", "CHAR(15)"),
+            # odps_types.Column("col18", "VARCHAR(15)"),
+            # odps_types.Column("col19", "decimal"),
+        ]
+    )
+    pd_dtypes = odps_schema_to_pandas_dtypes(odps_schema)
+    pd.testing.assert_series_equal(
+        pd_dtypes,
+        pd.Series(
+            [
+                np.dtype("O"),  # string
+                np.dtype("O"),  # binary
+                np.dtype(np.int8),
+                np.dtype(np.int16),
+                np.dtype(np.int32),
+                np.dtype(np.int64),
+                np.dtype(np.bool_),
+                np.dtype(np.float32),
+                np.dtype(np.float64),
+                np.dtype(
+                    "datetime64[ms]" if pd_release_version[0] >= 2 else "datetime64[ns]"
+                ),
+                np.dtype("datetime64[ns]"),
+                ArrowDtype(pa.list_(pa.string())),
+                ArrowDtype(pa.map_(pa.string(), pa.int64())),
+            ],
+            index=[c.name for c in odps_schema.columns],
+        ),
+    )
+
+    expected_odps_schema = odps_types.OdpsSchema(
+        [
+            odps_types.Column("col1", "string"),
+            odps_types.Column("col2", "string"),  # binary
+            odps_types.Column("col3", "tinyint"),
+            odps_types.Column("col4", "smallint"),
+            odps_types.Column("col5", "int"),
+            odps_types.Column("col6", "bigint"),
+            odps_types.Column("col7", "boolean"),
+            odps_types.Column("col8", "float"),
+            odps_types.Column("col9", "double"),
+            # odps_types.Column("col10", "date"),
+            odps_types.Column(
+                "col11", "datetime" if pd_release_version[0] >= 2 else "timestamp"
+            ),
+            odps_types.Column("col12", "timestamp"),
+            # odps_types.Column("col13", "decimal(10, 2)"),
+            odps_types.Column("col14", "array<string>"),
+            odps_types.Column("col15", "map<string, bigint>"),
+            # odps_types.Column("col16", "struct<a1: string, a2: map<string, bigint>>"),
+            # odps_types.Column("col17", "string"),
+            # odps_types.Column("col18", "string"),
+            # odps_types.Column("col19", "decimal(38, 18)"),
+        ]
+    )
+
+    odps_schema2 = arrow_schema_to_odps_schema(
+        pandas_dtypes_to_arrow_schema(pd_dtypes, unknown_as_string=True)
+    )
+    assert [c.name for c in expected_odps_schema.columns] == [
+        c.name for c in odps_schema2.columns
+    ]
+    assert [c.type for c in expected_odps_schema.columns] == [
+        c.type for c in odps_schema2.columns
+    ]
+
+    with pytest.raises(TypeError):
+        arrow_schema_to_odps_schema(pa.schema([("col1", pa.float16())]))
+
+
 def test_build_column_name():
     records = dict()
     assert build_table_column_name(0, "a" * 127, records) == "a" * 127
@@ -345,8 +456,10 @@ def test_pandas_types_to_arrow_schema():
         {
             "int8": pd.Series([], dtype=np.int8),
             "map": pd.Series([], dtype=dict_(pa.string(), pa.string())),
+            "list": pd.Series([], dtype=list_(pa.string())),
         },
     )
     schema = pandas_types_to_arrow_schema(pd_data)
     assert schema.field("int8").type == pa.int8()
     assert schema.field("map").type == pa.map_(pa.string(), pa.string())
+    assert schema.field("list").type == pa.list_(pa.string())

maxframe/io/odpsio/tests/test_tableio.py

@@ -14,15 +14,19 @@
 
 import datetime
 
+import mock
 import numpy as np
 import pandas as pd
 import pyarrow as pa
 import pytest
 from odps import ODPS
+from odps.errors import TableModified
+from odps.models import Table
 
 from ....config import options
 from ....tests.utils import flaky, tn
 from ....utils import config_odps_default_options
+from .. import TunnelTableIO
 from ..tableio import ODPSTableIO
 
 
@@ -161,3 +165,41 @@ def test_table_io_with_parts(switch_table_io):
             pd.testing.assert_frame_equal(reader.read_all().to_pandas(), expected_data)
     finally:
         tb.drop()
+
+
+def test_tunnel_table_io_with_modified():
+    config_odps_default_options()
+
+    o = ODPS.from_environments()
+    table_io = TunnelTableIO(o)
+
+    # test read and write tables with partition
+    parted_table_name = tn("test_tunnel_write_modified")
+    o.delete_table(parted_table_name, if_exists=True)
+    tb = o.create_table(
+        parted_table_name,
+        (",".join(f"{c} double" for c in "abcde"), "pt string"),
+        lifecycle=1,
+    )
+
+    raised = False
+    raw_open_reader = Table.open_reader
+
+    def _new_open_reader(self, *args, **kwargs):
+        nonlocal raised
+        if not raised:
+            raised = True
+            raise TableModified("Intentional error")
+        return raw_open_reader(self, *args, **kwargs)
+
+    try:
+        pd_data = pd.DataFrame(np.random.rand(100, 5), columns=list("abcde"))
+        with table_io.open_writer(parted_table_name, "pt=test") as writer:
+            writer.write(pa.Table.from_pandas(pd_data, preserve_index=False))
+
+        with mock.patch(
+            "odps.models.table.Table.open_reader", new=_new_open_reader
+        ), table_io.open_reader(parted_table_name, "pt=test") as reader:
+            pd.testing.assert_frame_equal(reader.read_all().to_pandas(), pd_data)
+    finally:
+        tb.drop()

maxframe/io/odpsio/tests/test_volumeio.py

@@ -12,79 +12,42 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import contextlib
+
 import pytest
 from odps import ODPS
 
-from ....tests.utils import tn
+from ....tests.utils import create_test_volume, get_test_unique_name, tn
 from ..volumeio import ODPSVolumeReader, ODPSVolumeWriter
 
 
 @pytest.fixture
 def create_volume(request, oss_config):
-    test_vol_name = tn("test_vol_name_" + request.param)
+    test_vol_name = tn(f"test_vol_name_{get_test_unique_name(5)}_" + request.param)
     odps_entry = ODPS.from_environments()
 
-    try:
-        odps_entry.delete_volume(test_vol_name)
-    except:
-        pass
-
-    oss_test_dir_name = None
-    if request.param == "parted":
-        odps_entry.create_parted_volume(test_vol_name)
-    else:
-        oss_test_dir_name = tn("test_oss_directory")
-        if oss_config is None:
-            pytest.skip("Need oss and its config to run this test")
-        (
-            oss_access_id,
-            oss_secret_access_key,
-            oss_bucket_name,
-            oss_endpoint,
-        ) = oss_config.oss_config
-
-        if "test" in oss_endpoint:
-            # offline config
-            test_location = "oss://%s:%s@%s/%s/%s" % (
-                oss_access_id,
-                oss_secret_access_key,
-                oss_endpoint,
-                oss_bucket_name,
-                oss_test_dir_name,
-            )
-            rolearn = None
-        else:
-            # online config
-            endpoint_parts = oss_endpoint.split(".", 1)
-            if "-internal" not in endpoint_parts[0]:
-                endpoint_parts[0] += "-internal"
-            test_location = "oss://%s/%s/%s" % (
-                ".".join(endpoint_parts),
-                oss_bucket_name,
-                oss_test_dir_name,
-            )
-            rolearn = oss_config.oss_rolearn
-
-        oss_config.oss_bucket.put_object(oss_test_dir_name + "/", b"")
-        odps_entry.create_external_volume(
-            test_vol_name, location=test_location, rolearn=rolearn
-        )
-    try:
-        yield test_vol_name
-    finally:
+    @contextlib.contextmanager
+    def create_parted_volume():
         try:
             odps_entry.delete_volume(test_vol_name)
-        except BaseException:
+        except:
             pass
+        try:
+            odps_entry.create_parted_volume(test_vol_name)
+            yield
+        finally:
+            try:
+                odps_entry.delete_volume(test_vol_name)
+            except BaseException:
+                pass
 
-        if oss_test_dir_name is not None:
-            import oss2
+    if request.param == "parted":
+        ctx = create_parted_volume()
+    else:
+        ctx = create_test_volume(test_vol_name, oss_config)
 
-            keys = [
-                obj.key
-                for obj in oss2.ObjectIterator(oss_config.oss_bucket, oss_test_dir_name)
-            ]
-            oss_config.oss_bucket.batch_delete_objects(keys)
+    with ctx:
+        yield test_vol_name
 
 
 @pytest.mark.parametrize("create_volume", ["external"], indirect=True)

maxframe/io/odpsio/volumeio.py

@@ -14,7 +14,9 @@
 
 import inspect
 from typing import Iterator, List, Optional, Union
+from urllib.parse import urlparse
 
+import requests
 from odps import ODPS
 from odps import __version__ as pyodps_version
 
@@ -74,14 +76,27 @@ class ODPSVolumeWriter:
         self._replace_internal_host = replace_internal_host
 
     def write_file(self, file_name: str, data: Union[bytes, Iterator[bytes]]):
-        kw = {}
-        if _has_replace_internal_host and self._replace_internal_host:
-            kw = {"replace_internal_host": self._replace_internal_host}
-        with self._volume.open_writer(
-            self._volume_dir + "/" + file_name, **kw
-        ) as writer:
+        sign_url = self._volume.get_sign_url(
+            self._volume_dir + "/" + file_name,
+            method="PUT",
+            seconds=3600,
+        )
+        if self._replace_internal_host:
+            parsed_url = urlparse(sign_url)
+            if "-internal." in parsed_url.netloc:
+                new_netloc = parsed_url.netloc.replace("-internal.", ".")
+                sign_url = sign_url.replace(parsed_url.netloc, new_netloc)
+
+        def _to_bytes(d):
+            if not isinstance(d, (bytes, bytearray)):
+                return bytes(d)
+            return d
+
+        def data_func():
             if not inspect.isgenerator(data):
-                writer.write(data)
+                yield _to_bytes(data)
             else:
                 for chunk in data:
-                    writer.write(chunk)
+                    yield _to_bytes(chunk)
+
+        requests.put(sign_url, data=data_func())

maxframe/learn/__init__.py

@@ -12,6 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from . import contrib
+from . import contrib, model_selection, preprocessing
 
-del contrib
+del contrib, model_selection, preprocessing

maxframe/learn/contrib/__init__.py

@@ -12,6 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from . import graph, llm, models, pytorch
+from . import graph, lightgbm, llm, pytorch
 
-del graph, llm, models, pytorch
+del graph, lightgbm, llm, pytorch

maxframe/learn/contrib/graph/connected_components.py

@@ -19,8 +19,9 @@ from maxframe import opcodes
 
 from ....core import OutputType
 from ....dataframe.operators import DataFrameOperator, DataFrameOperatorMixin
-from ....dataframe.utils import make_dtypes, parse_index
+from ....dataframe.utils import parse_index
 from ....serialization.serializables import Int32Field, StringField
+from ....utils import make_dtypes
 
 
 class DataFrameConnectedComponentsOperator(DataFrameOperator, DataFrameOperatorMixin):

maxframe/learn/contrib/lightgbm/__init__.py

@@ -0,0 +1,33 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ._predict import predict
+from ._train import train
+from .dataset import Dataset
+
+# isort: off
+from ..utils import config_mod_getattr as _config_mod_getattr
+
+_config_mod_getattr(
+    {
+        "LGBMClassifier": ".classifier.LGBMClassifier",
+        "LGBMRegressor": ".regressor.LGBMRegressor",
+        # "LGBMRanker": ".ranker.LGBMRanker",
+    },
+    globals(),
+)
+
+del _config_mod_getattr
+
+__all__ = ["Dataset", "LGBMClassifier", "LGBMRegressor", "predict", "train"]

maxframe/learn/contrib/lightgbm/_predict.py

@@ -0,0 +1,138 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Dict, List
+
+import numpy as np
+
+from .... import opcodes
+from ....core import OutputType
+from ....core.operator import Operator, TileableOperatorMixin
+from ....serialization.serializables import BoolField, DictField, Int32Field, KeyField
+from ....tensor.core import TensorOrder
+from ....typing_ import EntityType
+from ..models import to_remote_model
+from .dataset import check_data
+
+
+class LGBMPredict(Operator, TileableOperatorMixin):
+    _op_type_ = opcodes.LGBM_PREDICT
+    _op_module_ = "learn.contrib.lightgbm"
+    output_dtype = np.dtype(np.float64)
+
+    data = KeyField("data", default=None)
+    booster = KeyField("booster", default=None)
+    start_iteration = Int32Field("start_iteration", default=0)
+    num_iterations = Int32Field("num_iterations", default=None)
+    raw_score = BoolField("raw_score", default=False)
+    pred_leaf = BoolField("pred_leaf", default=False)
+    pred_contrib = BoolField("pred_contrib", default=False)
+    validate_features = BoolField("validate_features", default=False)
+    kwds = DictField("kwds", default=None)
+
+    def __init__(self, output_types=None, **kw):
+        super().__init__(_output_types=output_types, **kw)
+
+    def has_custom_code(self) -> bool:
+        return True
+
+    @property
+    def output_limit(self) -> int:
+        return 1 + self.pred_leaf + self.pred_contrib
+
+    @classmethod
+    def _set_inputs(cls, op: "LGBMPredict", inputs: List[EntityType]):
+        super()._set_inputs(op, inputs)
+        it = iter(inputs)
+        op.data = next(it)
+        op.booster = next(it)
+
+    def __call__(self):
+        num_class = getattr(self.booster.op, "num_class", None)
+
+        result_kw: Dict[str, Any] = {
+            "dtype": self.output_dtype,
+            "order": TensorOrder.C_ORDER,
+        }
+        kws = [result_kw]
+
+        if num_class is not None:
+            num_class = int(num_class)
+        if num_class is not None and num_class > 2:
+            result_kw["shape"] = (self.data.shape[0], num_class)
+        else:
+            result_kw["shape"] = (self.data.shape[0],)
+
+        if self.pred_leaf:
+            kws.append(
+                {
+                    "shape": (np.nan, np.nan),
+                    "dtype": np.dtype(np.float_),
+                    "order": TensorOrder.C_ORDER,
+                }
+            )
+        if self.pred_contrib:
+            kws.append(
+                {
+                    "shape": (np.nan, np.nan),
+                    "dtype": np.dtype(np.float_),
+                    "order": TensorOrder.C_ORDER,
+                }
+            )
+
+        return self.new_tileables([self.data, self.booster], kws=kws)
+
+
+def predict(
+    booster,
+    data,
+    raw_score: bool = False,
+    start_iteration: int = 0,
+    num_iteration: int = None,
+    pred_leaf: bool = False,
+    pred_contrib: bool = False,
+    validate_features: bool = False,
+    **kwargs,
+):
+    import lightgbm
+
+    from .core import Booster, BoosterData
+
+    if not isinstance(booster, (Booster, BoosterData, lightgbm.Booster)):
+        raise TypeError(
+            f"model has to be a lightgbm.Booster, got {type(booster)} instead"
+        )
+    elif isinstance(booster, lightgbm.Booster):
+        booster = to_remote_model(booster, model_cls=Booster)
+    proba = kwargs.pop("proba", False)
+
+    data = check_data(data)
+
+    op = LGBMPredict(
+        data=data,
+        booster=booster,
+        output_types=[OutputType.tensor],
+        proba=proba,
+        raw_score=raw_score,
+        start_iteration=start_iteration,
+        num_iteration=num_iteration,
+        pred_leaf=pred_leaf,
+        pred_contrib=pred_contrib,
+        validate_features=validate_features,
+        kwds=kwargs,
+    )
+    results = op()
+    if len(results) == 1:
+        return results[0]
+    return results