maxframe 1.0.0rc3__cp38-cp38-macosx_10_9_universal2.whl → 1.1.0__cp38-cp38-macosx_10_9_universal2.whl

maxframe/io/odpsio/tests/test_tableio.py

@@ -31,7 +31,7 @@ def switch_table_io(request):
     old_use_common_table = options.use_common_table
     try:
         options.use_common_table = request.param
-        yield
+        yield request.param
     finally:
         options.use_common_table = old_use_common_table
 
@@ -45,7 +45,7 @@ def test_empty_table_io(switch_table_io):
     table_io = ODPSTableIO(o)
 
     # test read from empty table
-    empty_table_name = tn("test_empty_table_halo_read")
+    empty_table_name = tn("test_empty_table_halo_read_" + str(switch_table_io).lower())
     o.delete_table(empty_table_name, if_exists=True)
     tb = o.create_table(empty_table_name, "col1 string", lifecycle=1)
 
@@ -65,7 +65,7 @@ def test_table_io_without_parts(switch_table_io):
     table_io = ODPSTableIO(o)
 
     # test read and write tables without partition
-    no_part_table_name = tn("test_no_part_halo_write")
+    no_part_table_name = tn("test_no_part_halo_write_" + str(switch_table_io).lower())
     o.delete_table(no_part_table_name, if_exists=True)
     col_desc = ",".join(f"{c} double" for c in "abcde") + ", f datetime"
     tb = o.create_table(no_part_table_name, col_desc, lifecycle=1)
@@ -99,7 +99,7 @@ def test_table_io_with_range_reader(switch_table_io):
     table_io = ODPSTableIO(o)
 
     # test read and write tables without partition
-    no_part_table_name = tn("test_no_part_halo_write")
+    no_part_table_name = tn("test_halo_write_range_" + str(switch_table_io).lower())
     o.delete_table(no_part_table_name, if_exists=True)
     tb = o.create_table(
         no_part_table_name, ",".join(f"{c} double" for c in "abcde"), lifecycle=1
@@ -139,7 +139,7 @@ def test_table_io_with_parts(switch_table_io):
     table_io = ODPSTableIO(o)
 
     # test read and write tables with partition
-    parted_table_name = tn("test_parted_halo_write")
+    parted_table_name = tn("test_parted_halo_write_" + str(switch_table_io).lower())
     o.delete_table(parted_table_name, if_exists=True)
     tb = o.create_table(
         parted_table_name,

maxframe/io/odpsio/tests/test_volumeio.py

@@ -42,15 +42,33 @@ def create_volume(request, oss_config):
             oss_bucket_name,
             oss_endpoint,
         ) = oss_config.oss_config
-        test_location = "oss://%s:%s@%s/%s/%s" % (
-            oss_access_id,
-            oss_secret_access_key,
-            oss_endpoint,
-            oss_bucket_name,
-            oss_test_dir_name,
-        )
+
+        if "test" in oss_endpoint:
+            # offline config
+            test_location = "oss://%s:%s@%s/%s/%s" % (
+                oss_access_id,
+                oss_secret_access_key,
+                oss_endpoint,
+                oss_bucket_name,
+                oss_test_dir_name,
+            )
+            rolearn = None
+        else:
+            # online config
+            endpoint_parts = oss_endpoint.split(".", 1)
+            if "-internal" not in endpoint_parts[0]:
+                endpoint_parts[0] += "-internal"
+            test_location = "oss://%s/%s/%s" % (
+                ".".join(endpoint_parts),
+                oss_bucket_name,
+                oss_test_dir_name,
+            )
+            rolearn = oss_config.oss_rolearn
+
         oss_config.oss_bucket.put_object(oss_test_dir_name + "/", b"")
-        odps_entry.create_external_volume(test_vol_name, location=test_location)
+        odps_entry.create_external_volume(
+            test_vol_name, location=test_location, rolearn=rolearn
+        )
     try:
         yield test_vol_name
     finally:
@@ -75,13 +93,19 @@ def test_read_write_volume(create_volume):
 
     odps_entry = ODPS.from_environments()
 
-    writer = ODPSVolumeWriter(odps_entry, create_volume, test_vol_dir)
+    writer = ODPSVolumeWriter(
+        odps_entry, create_volume, test_vol_dir, replace_internal_host=True
+    )
 
-    writer = ODPSVolumeWriter(odps_entry, create_volume, test_vol_dir)
+    writer = ODPSVolumeWriter(
+        odps_entry, create_volume, test_vol_dir, replace_internal_host=True
+    )
     writer.write_file("file1", b"content1")
     writer.write_file("file2", b"content2")
 
-    reader = ODPSVolumeReader(odps_entry, create_volume, test_vol_dir)
+    reader = ODPSVolumeReader(
+        odps_entry, create_volume, test_vol_dir, replace_internal_host=True
+    )
     assert reader.read_file("file1") == b"content1"
     assert reader.read_file("file2") == b"content2"
 

maxframe/io/odpsio/volumeio.py

@@ -13,16 +13,28 @@
 # limitations under the License.
 
 import inspect
-from typing import Iterator, List, Union
+from typing import Iterator, List, Optional, Union
 
 from odps import ODPS
+from odps import __version__ as pyodps_version
+
+from ...lib.version import Version
+
+_has_replace_internal_host = Version(pyodps_version) >= Version("0.12.0")
 
 
 class ODPSVolumeReader:
-    def __init__(self, odps_entry: ODPS, volume_name: str, volume_dir: str):
+    def __init__(
+        self,
+        odps_entry: ODPS,
+        volume_name: str,
+        volume_dir: str,
+        replace_internal_host: bool = False,
+    ):
         self._odps_entry = odps_entry
         self._volume = odps_entry.get_volume(volume_name)
         self._volume_dir = volume_dir
+        self._replace_internal_host = replace_internal_host
 
     def list_files(self) -> List[str]:
         def _get_file_name(vol_file):
@@ -38,18 +50,36 @@ class ODPSVolumeReader:
         ]
 
     def read_file(self, file_name: str) -> bytes:
-        with self._volume.open_reader(self._volume_dir + "/" + file_name) as reader:
+        kw = {}
+        if _has_replace_internal_host and self._replace_internal_host:
+            kw = {"replace_internal_host": self._replace_internal_host}
+        with self._volume.open_reader(
+            self._volume_dir + "/" + file_name, **kw
+        ) as reader:
             return reader.read()
 
 
 class ODPSVolumeWriter:
-    def __init__(self, odps_entry: ODPS, volume_name: str, volume_dir: str):
+    def __init__(
+        self,
+        odps_entry: ODPS,
+        volume_name: str,
+        volume_dir: str,
+        schema_name: Optional[str] = None,
+        replace_internal_host: bool = False,
+    ):
         self._odps_entry = odps_entry
-        self._volume = odps_entry.get_volume(volume_name)
+        self._volume = odps_entry.get_volume(volume_name, schema=schema_name)
         self._volume_dir = volume_dir
+        self._replace_internal_host = replace_internal_host
 
     def write_file(self, file_name: str, data: Union[bytes, Iterator[bytes]]):
-        with self._volume.open_writer(self._volume_dir + "/" + file_name) as writer:
+        kw = {}
+        if _has_replace_internal_host and self._replace_internal_host:
+            kw = {"replace_internal_host": self._replace_internal_host}
+        with self._volume.open_writer(
+            self._volume_dir + "/" + file_name, **kw
+        ) as writer:
             if not inspect.isgenerator(data):
                 writer.write(data)
             else:

maxframe/learn/contrib/__init__.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from . import pytorch
+from . import graph, llm, pytorch
 
+del graph
+del llm
 del pytorch

maxframe/learn/contrib/graph/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .connected_components import connected_components

maxframe/learn/contrib/graph/connected_components.py

@@ -0,0 +1,215 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+import pandas as pd
+
+from maxframe import opcodes
+
+from ....core import OutputType
+from ....dataframe.operators import DataFrameOperator, DataFrameOperatorMixin
+from ....dataframe.utils import make_dtypes, parse_index
+from ....serialization.serializables import Int32Field, StringField
+
+
+class DataFrameConnectedComponentsOperator(DataFrameOperator, DataFrameOperatorMixin):
+    _op_type_ = opcodes.CONNECTED_COMPONENTS
+
+    vertex_col1 = StringField("vertex_col1", default=None)
+    vertex_col2 = StringField("vertex_col2", default=None)
+    max_iter = Int32Field("max_iter", default=6)
+
+    def __call__(self, df):
+        node_id_dtype = df.dtypes[self.vertex_col1]
+        dtypes = make_dtypes({"id": node_id_dtype, "component": node_id_dtype})
+        # this will return a dataframe and a bool flag
+        new_dataframe_tileable_kw = {
+            "shape": (np.nan, 2),
+            "index_value": parse_index(pd.RangeIndex(0)),
+            "columns_value": parse_index(dtypes.index, store_data=True),
+            "dtypes": dtypes,
+        }
+        new_scalar_tileable_kw = {"dtype": np.dtype(np.bool_), "shape": ()}
+        return self.new_tileables(
+            [df],
+            kws=[new_dataframe_tileable_kw, new_scalar_tileable_kw],
+        )
+
+    @property
+    def output_limit(self):
+        return 2
+
+
+def connected_components(
+    dataframe, vertex_col1: str, vertex_col2: str, max_iter: int = 6
+):
+    """
+    The connected components algorithm labels each node as belonging to a specific connected component with the ID of
+    its lowest-numbered vertex.
+
+    Parameters
+    ----------
+    dataframe : DataFrame
+        A DataFrame containing the edges of the graph.
+
+    vertex_col1 : str
+        The name of the column in `dataframe` that contains the one of edge vertices. The column value must be an
+        integer.
+
+    vertex_col2 : str
+        The name of the column in `dataframe` that contains the other one of edge vertices. The column value must be an
+        integer.
+
+    max_iter : int
+        The algorithm use large and small star transformation to find all connected components, `max_iter`
+        controls the max round of the iterations before finds all edges. Default is 6.
+
+
+    Returns
+    -------
+    DataFrame
+        Return dataFrame contains all connected component edges by two columns `id` and `component`. `component` is
+        the lowest-numbered vertex in the connected components.
+
+    Notes
+    -------
+    After `execute()`, the dataframe has a bool member `flag` to indicate if the `connected_components` already
+    converged in `max_iter` rounds. `True` means the dataframe already contains all edges of the connected components.
+    If `False` you can run `connected_components` more times to reach the converged state.
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> import maxframe.dataframe as md
+    >>> import maxframe.learn.contrib.graph.connected_components
+    >>> df = md.DataFrame({'x': [4, 1], 'y': [0, 4]})
+    >>> df.execute()
+       x  y
+    0  4  1
+    1  0  4
+
+    Get connected components with 1 round iteration.
+
+    >>> components, converged = connected_components(df, "x", "y", 1)
+    >>> session.execute(components, converged)
+    >>> components
+       A   B
+    0  1   0
+    1  4   0
+
+    >>> converged
+    True
+
+    Sometimes, a single iteration may not be sufficient to propagate the connectivity of all edges.
+    By default, `connected_components` performs 6 iterations of calculations.
+    If you are unsure whether the connected components have converged, you can check the `flag` variable in
+    the output DataFrame after calling `execute()`.
+
+    >>> df = md.DataFrame({'x': [4, 1, 7, 5, 8, 11, 11], 'y': [0, 4, 4, 7, 7, 9, 13]})
+    >>> df.execute()
+        x   y
+    0   4   0
+    1   1   4
+    2   7   4
+    3   5   7
+    4   8   7
+    5  11   9
+    6  11  13
+
+    >>> components, converged = connected_components(df, "x", "y", 1)
+    >>> session.execute(components, converged)
+    >>> components
+       id  component
+    0   4          0
+    1   7          0
+    2   8          4
+    3  13          9
+    4   1          0
+    5   5          0
+    6  11          9
+
+    If `flag` is True, it means convergence has been achieved.
+
+    >>> converged
+    False
+
+    You can determine whether to continue iterating or to use a larger number of iterations
+    (but not too large, which would result in wasted computational overhead).
+
+    >>> components, converged = connected_components(components, "id", "component", 1)
+    >>> session.execute(components, converged)
+    >>> components
+       id  component
+    0   4          0
+    1   7          0
+    2  13          9
+    3   1          0
+    4   5          0
+    5  11          9
+    6   8          0
+
+    >>> components, converged = connected_components(df, "x", "y")
+    >>> session.execute(components, converged)
+    >>> components
+       id  component
+    0   4          0
+    1   7          0
+    2  13          9
+    3   1          0
+    4   5          0
+    5  11          9
+    6   8          0
+    """
+
+    # Check if vertex columns are provided
+    if not vertex_col1 or not vertex_col2:
+        raise ValueError("Both vertex_col1 and vertex_col2 must be provided.")
+
+    # Check if max_iter is provided and within the valid range
+    if max_iter is None:
+        raise ValueError("max_iter must be provided.")
+    if not (1 <= max_iter <= 50):
+        raise ValueError("max_iter must be an integer between 1 and 50.")
+
+    # Verify that the vertex columns exist in the dataframe
+    missing_cols = [
+        col for col in (vertex_col1, vertex_col2) if col not in dataframe.dtypes
+    ]
+    if missing_cols:
+        raise ValueError(
+            f"The following required columns {missing_cols} are not in {list(dataframe.dtypes.index)}"
+        )
+
+    # Ensure that the vertex columns are of integer type
+    # TODO support string dtype
+    incorrect_dtypes = [
+        col
+        for col in (vertex_col1, vertex_col2)
+        if dataframe[col].dtype != np.dtype("int")
+    ]
+    if incorrect_dtypes:
+        dtypes_str = ", ".join(str(dataframe[col].dtype) for col in incorrect_dtypes)
+        raise ValueError(
+            f"Columns {incorrect_dtypes} should be of integer type, but found {dtypes_str}."
+        )
+
+    op = DataFrameConnectedComponentsOperator(
+        vertex_col1=vertex_col1,
+        vertex_col2=vertex_col2,
+        _output_types=[OutputType.dataframe, OutputType.scalar],
+        max_iter=max_iter,
+    )
+    return op(
+        dataframe,
+    )

maxframe/learn/contrib/graph/tests/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

maxframe/learn/contrib/graph/tests/test_connected_components.py

@@ -0,0 +1,53 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+import pytest
+
+from ..... import dataframe as md
+from .....dataframe.core import DataFrameData
+from .....tensor.core import TensorData
+from .. import connected_components
+
+
+@pytest.fixture
+def df1():
+    return md.DataFrame({"a": [1, 2, 3], "b": [1, 2, 3], "c": [1, 2, 3]})
+
+
+@pytest.fixture
+def df2():
+    return md.DataFrame(
+        [[1, "2"], [1, "2"]],
+        columns=["a", "b"],
+    )
+
+
+def test_connected_components(df1, df2):
+    edges, flag = connected_components(df1, "a", "b")
+    assert edges.op.max_iter == 6
+    assert edges.shape == (np.nan, 2)
+    assert isinstance(edges.data, DataFrameData)
+    assert isinstance(flag.data, TensorData)
+    assert flag.shape == ()
+    assert "id" in edges.dtypes and "component" in edges.dtypes
+
+    with pytest.raises(ValueError):
+        connected_components(df1, "a", "x")
+
+    with pytest.raises(ValueError):
+        connected_components(df1, "a", "b", 0)
+
+    with pytest.raises(ValueError):
+        connected_components(df2, "a", "b")

maxframe/learn/contrib/llm/__init__.py

@@ -0,0 +1,16 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from . import models, multi_modal, text
+
+del models

maxframe/learn/contrib/llm/core.py

@@ -0,0 +1,54 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import Any, Dict
+
+import numpy as np
+import pandas as pd
+
+from ....core.entity.output_types import OutputType
+from ....core.operator.base import Operator
+from ....core.operator.core import TileableOperatorMixin
+from ....dataframe.utils import parse_index
+from ....serialization.serializables.core import Serializable
+from ....serialization.serializables.field import AnyField, DictField, StringField
+
+
+class LLM(Serializable):
+    name = StringField("name", default=None)
+
+    def validate_params(self, params: Dict[str, Any]):
+        pass
+
+
+class LLMOperator(Operator, TileableOperatorMixin):
+    model = AnyField("model", default=None)
+    prompt_template = AnyField("prompt_template", default=None)
+    params = DictField("params", default=None)
+
+    def __init__(self, output_types=None, **kw):
+        if output_types is None:
+            output_types = [OutputType.dataframe]
+        super().__init__(_output_types=output_types, **kw)
+
+    def __call__(self, data):
+        col_names = ["response", "success"]
+        columns = parse_index(pd.Index(col_names), store_data=True)
+        out_dtypes = pd.Series([np.dtype("O"), np.dtype("bool")], index=col_names)
+        return self.new_tileable(
+            inputs=[data],
+            dtypes=out_dtypes,
+            shape=(data.shape[0], len(col_names)),
+            index_value=data.index_value,
+            columns_value=columns,
+        )

maxframe/learn/contrib/llm/models/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from .dashscope import DashScopeMultiModalLLM, DashScopeTextLLM

maxframe/learn/contrib/llm/models/dashscope.py

@@ -0,0 +1,73 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import Any, Dict
+
+from ..... import opcodes
+from .....serialization.serializables.core import Serializable
+from .....serialization.serializables.field import StringField
+from ..core import LLMOperator
+from ..multi_modal import MultiModalLLM
+from ..text import TextLLM
+
+
+class DashScopeLLMMixin(Serializable):
+    __slots__ = ()
+
+    _not_supported_params = {"stream", "incremental_output"}
+
+    def validate_params(self, params: Dict[str, Any]):
+        for k in params.keys():
+            if k in self._not_supported_params:
+                raise ValueError(f"{k} is not supported")
+
+
+class DashScopeTextLLM(TextLLM, DashScopeLLMMixin):
+    api_key_resource = StringField("api_key_resource", default=None)
+
+    def generate(
+        self,
+        data,
+        prompt_template: Dict[str, Any],
+        params: Dict[str, Any] = None,
+    ):
+        return DashScopeTextGenerationOperator(
+            model=self,
+            prompt_template=prompt_template,
+            params=params,
+        )(data)
+
+
+class DashScopeMultiModalLLM(MultiModalLLM, DashScopeLLMMixin):
+    api_key_resource = StringField("api_key_resource", default=None)
+
+    def generate(
+        self,
+        data,
+        prompt_template: Dict[str, Any],
+        params: Dict[str, Any] = None,
+    ):
+        # TODO add precheck here
+        return DashScopeMultiModalGenerationOperator(
+            model=self,
+            prompt_template=prompt_template,
+            params=params,
+        )(data)
+
+
+class DashScopeTextGenerationOperator(LLMOperator):
+    _op_type_ = opcodes.DASHSCOPE_TEXT_GENERATION
+
+
+class DashScopeMultiModalGenerationOperator(LLMOperator):
+    _op_type_ = opcodes.DASHSCOPE_MULTI_MODAL_GENERATION