maxframe 0.1.0b4__cp38-cp38-macosx_10_9_universal2.whl → 1.0.0__cp38-cp38-macosx_10_9_universal2.whl

maxframe/io/odpsio/tests/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

maxframe/{odpsio → io/odpsio}/tests/test_schema.py

@@ -18,9 +18,10 @@ import pyarrow as pa
 import pytest
 from odps import types as odps_types
 
-from ... import dataframe as md
-from ... import tensor as mt
-from ...core import OutputType
+from .... import dataframe as md
+from .... import tensor as mt
+from ....core import OutputType
+from ....utils import pd_release_version
 from ..schema import (
     arrow_schema_to_odps_schema,
     build_dataframe_table_meta,
@@ -30,20 +31,23 @@ from ..schema import (
 )
 
 
-def _wrap_maxframe_obj(obj, wrap=True):
-    if not wrap:
+def _wrap_maxframe_obj(obj, wrap="no"):
+    if wrap == "no":
         return obj
     if isinstance(obj, pd.DataFrame):
-        return md.DataFrame(obj)
+        obj = md.DataFrame(obj)
     elif isinstance(obj, pd.Series):
-        return md.Series(obj)
+        obj = md.Series(obj)
     elif isinstance(obj, pd.Index):
-        return md.Index(obj)
+        obj = md.Index(obj)
     else:
-        return mt.scalar(obj)
+        obj = mt.scalar(obj)
+    if wrap == "data":
+        return obj.data
+    return obj
 
 
-@pytest.mark.parametrize("wrap_obj", [False, True])
+@pytest.mark.parametrize("wrap_obj", ["no", "yes", "data"])
 def test_pandas_to_odps_schema_dataframe(wrap_obj):
     data = pd.DataFrame(np.random.rand(100, 5), columns=list("ABCDE"))
 
@@ -94,7 +98,7 @@ def test_pandas_to_odps_schema_dataframe(wrap_obj):
     assert meta.pd_index_level_names == [None, None]
 
 
-@pytest.mark.parametrize("wrap_obj", [False, True])
+@pytest.mark.parametrize("wrap_obj", ["no", "yes", "data"])
 def test_pandas_to_odps_schema_series(wrap_obj):
     data = pd.Series(np.random.rand(100))
 
@@ -135,22 +139,22 @@ def test_pandas_to_odps_schema_series(wrap_obj):
     assert meta.pd_index_level_names == ["c1", "c2"]
 
 
-@pytest.mark.parametrize("wrap_obj", [False, True])
+@pytest.mark.parametrize("wrap_obj", ["no", "yes", "data"])
 def test_pandas_to_odps_schema_index(wrap_obj):
     data = pd.Index(np.random.randint(0, 100, 100))
 
     test_idx = _wrap_maxframe_obj(data, wrap=wrap_obj)
-    schema, meta = pandas_to_odps_schema(test_idx, unknown_as_string=True)
-    assert [c.name for c in schema.columns] == ["_idx_0"]
-    assert [c.type.name for c in schema.columns] == ["bigint"]
-    assert meta.type == OutputType.index
-    assert meta.table_column_names == []
-    assert meta.table_index_column_names == ["_idx_0"]
-    assert meta.pd_column_level_names == []
-    assert meta.pd_index_level_names == [None]
-
-    with pytest.raises(AssertionError):
-        pandas_to_odps_schema(test_idx, unknown_as_string=True, ignore_index=True)
+    for ignore_idx in (False, True):
+        schema, meta = pandas_to_odps_schema(
+            test_idx, unknown_as_string=True, ignore_index=ignore_idx
+        )
+        assert [c.name for c in schema.columns] == ["_idx_0"]
+        assert [c.type.name for c in schema.columns] == ["bigint"]
+        assert meta.type == OutputType.index
+        assert meta.table_column_names == []
+        assert meta.table_index_column_names == ["_idx_0"]
+        assert meta.pd_column_level_names == []
+        assert meta.pd_index_level_names == [None]
 
     data = pd.MultiIndex.from_arrays(
         [np.random.choice(list("ABC"), 100), np.random.randint(0, 10, 100)],
@@ -167,11 +171,14 @@ def test_pandas_to_odps_schema_index(wrap_obj):
     assert meta.pd_index_level_names == ["c1", "c2"]
 
 
-@pytest.mark.parametrize("wrap_obj", [False, True])
+@pytest.mark.parametrize("wrap_obj", ["no", "yes", "data"])
 def test_pandas_to_odps_schema_scalar(wrap_obj):
     data = 1234.56
 
     test_scalar = _wrap_maxframe_obj(data, wrap=wrap_obj)
+    if wrap_obj != "no":
+        test_scalar.op.data = None
+
     schema, meta = pandas_to_odps_schema(test_scalar, unknown_as_string=True)
     assert schema.columns[0].name == "_idx_0"
     assert schema.columns[0].type.name == "double"
@@ -181,9 +188,6 @@ def test_pandas_to_odps_schema_scalar(wrap_obj):
     assert meta.pd_column_level_names == []
     assert meta.pd_index_level_names == [None]
 
-    with pytest.raises(AssertionError):
-        pandas_to_odps_schema(test_scalar, unknown_as_string=True, ignore_index=True)
-
 
 def test_odps_arrow_schema_conversion():
     odps_schema = odps_types.OdpsSchema(
@@ -206,10 +210,11 @@ def test_odps_arrow_schema_conversion():
             odps_types.Column("col16", "struct<a1: string, a2: map<string, bigint>>"),
             odps_types.Column("col17", "CHAR(15)"),
             odps_types.Column("col18", "VARCHAR(15)"),
+            odps_types.Column("col19", "decimal"),
         ]
     )
     arrow_schema = odps_schema_to_arrow_schema(odps_schema)
-    assert arrow_schema.names == [f"col{i}" for i in range(1, 19)]
+    assert arrow_schema.names == [f"col{i}" for i in range(1, 20)]
     assert arrow_schema.types == [
         pa.string(),
         pa.binary(),
@@ -229,6 +234,7 @@ def test_odps_arrow_schema_conversion():
         pa.struct([("a1", pa.string()), ("a2", pa.map_(pa.string(), pa.int64()))]),
         pa.string(),
         pa.string(),
+        pa.decimal128(38, 18),
     ]
 
     expected_odps_schema = odps_types.OdpsSchema(
@@ -251,6 +257,7 @@ def test_odps_arrow_schema_conversion():
             odps_types.Column("col16", "struct<a1: string, a2: map<string, bigint>>"),
             odps_types.Column("col17", "string"),
             odps_types.Column("col18", "string"),
+            odps_types.Column("col19", "decimal(38, 18)"),
         ]
     )
 
@@ -264,10 +271,6 @@ def test_odps_arrow_schema_conversion():
 
     with pytest.raises(TypeError):
         arrow_schema_to_odps_schema(pa.schema([("col1", pa.float16())]))
-    with pytest.raises(TypeError):
-        odps_schema_to_arrow_schema(
-            odps_types.OdpsSchema([odps_types.Column("col1", "json")])
-        )
 
 
 def test_build_column_name():
@@ -279,7 +282,7 @@ def test_build_column_name():
     assert build_table_column_name(4, ("A", 1), records) == "a_1"
 
 
-@pytest.mark.parametrize("wrap_obj", [False, True])
+@pytest.mark.parametrize("wrap_obj", ["no", "yes", "data"])
 def test_build_table_meta(wrap_obj):
     data = pd.DataFrame(
         np.random.rand(100, 7),
@@ -290,3 +293,42 @@ def test_build_table_meta(wrap_obj):
     table_meta = build_dataframe_table_meta(test_df)
     expected_cols = ["a_2", "a_3", "a_0", "a_1_0", "a_1_1", "b", "c"]
     assert table_meta.table_column_names == expected_cols
+
+
+@pytest.mark.skipif(
+    pd_release_version[0] < 2, reason="only run under pandas 2.0 or greater"
+)
+def test_table_meta_with_datetime():
+    raw_df = pd.DataFrame(
+        [
+            [1, "abc", "2024-10-01 11:23:12"],
+            [3, "uvw", "2024-10-02 22:55:13"],
+        ],
+        columns=["col1", "col2", "col3"],
+    )
+    df = md.DataFrame(raw_df).astype({"col3": "datetime64[ms]"})
+    schema, _ = pandas_to_odps_schema(df, unknown_as_string=True)
+    assert schema.columns[3].type == odps_types.datetime
+
+    raw_series = pd.Series(
+        ["2024-10-01 11:23:12", "2024-10-02 22:55:13"], dtype="datetime64[ms]"
+    )
+    s = md.Series(raw_series)
+    schema, _ = pandas_to_odps_schema(s, unknown_as_string=True)
+    assert schema.columns[1].type == odps_types.datetime
+
+    raw_index = pd.Index(
+        ["2024-10-01 11:23:12", "2024-10-02 22:55:13"], dtype="datetime64[ms]"
+    )
+    idx = md.Index(raw_index)
+    schema, _ = pandas_to_odps_schema(idx, unknown_as_string=True)
+    assert schema.columns[0].type == odps_types.datetime
+
+    src_df = pd.DataFrame(
+        [[1, "2024-10-01 11:23:12"], [3, "2024-10-02 22:55:13"]],
+        columns=["A", "B"],
+    ).astype({"B": "datetime64[ms]"})
+    raw_multiindex = pd.MultiIndex.from_frame(src_df)
+    multiidx = md.Index(raw_multiindex)
+    schema, _ = pandas_to_odps_schema(multiidx, unknown_as_string=True)
+    assert schema.columns[1].type == odps_types.datetime

maxframe/{odpsio → io/odpsio}/tests/test_tableio.py

@@ -12,22 +12,37 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import datetime
+
 import numpy as np
 import pandas as pd
 import pyarrow as pa
+import pytest
 from odps import ODPS
 
-from ...tests.utils import flaky, tn
-from ...utils import config_odps_default_options
-from ..tableio import HaloTableIO
+from ....config import options
+from ....tests.utils import flaky, tn
+from ....utils import config_odps_default_options
+from ..tableio import ODPSTableIO
+
+
+@pytest.fixture
+def switch_table_io(request):
+    old_use_common_table = options.use_common_table
+    try:
+        options.use_common_table = request.param
+        yield
+    finally:
+        options.use_common_table = old_use_common_table
 
 
 @flaky(max_runs=3)
-def test_empty_table_io():
+@pytest.mark.parametrize("switch_table_io", [False, True], indirect=True)
+def test_empty_table_io(switch_table_io):
     config_odps_default_options()
 
     o = ODPS.from_environments()
-    halo_table_io = HaloTableIO(o)
+    table_io = ODPSTableIO(o)
 
     # test read from empty table
     empty_table_name = tn("test_empty_table_halo_read")
@@ -35,42 +50,53 @@ def test_empty_table_io():
     tb = o.create_table(empty_table_name, "col1 string", lifecycle=1)
 
     try:
-        with halo_table_io.open_reader(empty_table_name) as reader:
+        with table_io.open_reader(empty_table_name) as reader:
             assert len(reader.read_all()) == 0
     finally:
         tb.drop()
 
 
 @flaky(max_runs=3)
-def test_table_io_without_parts():
+@pytest.mark.parametrize("switch_table_io", [False, True], indirect=True)
+def test_table_io_without_parts(switch_table_io):
     config_odps_default_options()
 
     o = ODPS.from_environments()
-    halo_table_io = HaloTableIO(o)
+    table_io = ODPSTableIO(o)
 
     # test read and write tables without partition
     no_part_table_name = tn("test_no_part_halo_write")
     o.delete_table(no_part_table_name, if_exists=True)
-    tb = o.create_table(
-        no_part_table_name, ",".join(f"{c} double" for c in "abcde"), lifecycle=1
-    )
+    col_desc = ",".join(f"{c} double" for c in "abcde") + ", f datetime"
+    tb = o.create_table(no_part_table_name, col_desc, lifecycle=1)
 
     try:
         pd_data = pd.DataFrame(np.random.rand(100, 5), columns=list("abcde"))
-        with halo_table_io.open_writer(no_part_table_name) as writer:
+        date_val = [
+            (
+                datetime.datetime.now().replace(microsecond=0)
+                + datetime.timedelta(seconds=i)
+            )
+            for i in range(100)
+        ]
+        pd_data["f"] = pd.Series(date_val, dtype="datetime64[ms]").dt.tz_localize(
+            options.local_timezone
+        )
+        with table_io.open_writer(no_part_table_name) as writer:
             writer.write(pa.Table.from_pandas(pd_data, preserve_index=False))
-        with halo_table_io.open_reader(no_part_table_name) as reader:
+        with table_io.open_reader(no_part_table_name) as reader:
             pd.testing.assert_frame_equal(reader.read_all().to_pandas(), pd_data)
     finally:
         tb.drop()
 
 
 @flaky(max_runs=3)
-def test_table_io_with_range_reader():
+@pytest.mark.parametrize("switch_table_io", [False, True], indirect=True)
+def test_table_io_with_range_reader(switch_table_io):
     config_odps_default_options()
 
     o = ODPS.from_environments()
-    halo_table_io = HaloTableIO(o)
+    table_io = ODPSTableIO(o)
 
     # test read and write tables without partition
     no_part_table_name = tn("test_no_part_halo_write")
@@ -81,15 +107,15 @@ def test_table_io_with_range_reader():
 
     try:
         pd_data = pd.DataFrame(np.random.rand(100, 5), columns=list("abcde"))
-        with halo_table_io.open_writer(no_part_table_name) as writer:
+        with table_io.open_writer(no_part_table_name) as writer:
             writer.write(pa.Table.from_pandas(pd_data, preserve_index=False))
 
-        with halo_table_io.open_reader(
+        with table_io.open_reader(
             no_part_table_name, start=None, stop=100, row_batch_size=10
         ) as reader:
             pd.testing.assert_frame_equal(reader.read_all().to_pandas(), pd_data)
 
-        with halo_table_io.open_reader(
+        with table_io.open_reader(
             no_part_table_name,
             start=-2,
             stop=-52,
@@ -105,11 +131,12 @@ def test_table_io_with_range_reader():
 
 
 @flaky(max_runs=3)
-def test_table_io_with_parts():
+@pytest.mark.parametrize("switch_table_io", [False, True], indirect=True)
+def test_table_io_with_parts(switch_table_io):
     config_odps_default_options()
 
     o = ODPS.from_environments()
-    halo_table_io = HaloTableIO(o)
+    table_io = ODPSTableIO(o)
 
     # test read and write tables with partition
     parted_table_name = tn("test_parted_halo_write")
@@ -122,11 +149,11 @@ def test_table_io_with_parts():
 
     try:
         pd_data = pd.DataFrame(np.random.rand(100, 5), columns=list("abcde"))
-        with halo_table_io.open_writer(parted_table_name, "pt=test") as writer:
+        with table_io.open_writer(parted_table_name, "pt=test") as writer:
             writer.write(pa.Table.from_pandas(pd_data, preserve_index=False))
-        with halo_table_io.open_reader(parted_table_name, "pt=test") as reader:
+        with table_io.open_reader(parted_table_name, "pt=test") as reader:
             pd.testing.assert_frame_equal(reader.read_all().to_pandas(), pd_data)
-        with halo_table_io.open_reader(
+        with table_io.open_reader(
             parted_table_name, "pt=test", partition_columns=True
         ) as reader:
             expected_data = pd_data.copy()

maxframe/{odpsio → io/odpsio}/tests/test_volumeio.py

@@ -15,7 +15,7 @@
 import pytest
 from odps import ODPS
 
-from ...tests.utils import tn
+from ....tests.utils import tn
 from ..volumeio import ODPSVolumeReader, ODPSVolumeWriter
 
 
@@ -69,19 +69,17 @@ def create_volume(request, oss_config):
             oss_config.oss_bucket.batch_delete_objects(keys)
 
 
-@pytest.mark.parametrize("create_volume", ["parted", "external"], indirect=True)
+@pytest.mark.parametrize("create_volume", ["external"], indirect=True)
 def test_read_write_volume(create_volume):
     test_vol_dir = "test_vol_dir"
 
     odps_entry = ODPS.from_environments()
 
     writer = ODPSVolumeWriter(odps_entry, create_volume, test_vol_dir)
-    write_session_id = writer.create_write_session()
 
     writer = ODPSVolumeWriter(odps_entry, create_volume, test_vol_dir)
-    writer.write_file("file1", b"content1", write_session_id)
-    writer.write_file("file2", b"content2", write_session_id)
-    writer.commit(["file1", "file2"], write_session_id)
+    writer.write_file("file1", b"content1")
+    writer.write_file("file2", b"content2")
 
     reader = ODPSVolumeReader(odps_entry, create_volume, test_vol_dir)
     assert reader.read_file("file1") == b"content1"

maxframe/io/odpsio/volumeio.py

@@ -0,0 +1,63 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import inspect
+from typing import Iterator, List, Optional, Union
+
+from odps import ODPS
+
+
+class ODPSVolumeReader:
+    def __init__(self, odps_entry: ODPS, volume_name: str, volume_dir: str):
+        self._odps_entry = odps_entry
+        self._volume = odps_entry.get_volume(volume_name)
+        self._volume_dir = volume_dir
+
+    def list_files(self) -> List[str]:
+        def _get_file_name(vol_file):
+            if hasattr(vol_file, "name"):
+                return vol_file.name
+            return vol_file.path.rsplit("/", 1)[-1]
+
+        return [
+            _get_file_name(f)
+            for f in self._odps_entry.list_volume_files(
+                f"/{self._volume.name}/{self._volume_dir}"
+            )
+        ]
+
+    def read_file(self, file_name: str) -> bytes:
+        with self._volume.open_reader(self._volume_dir + "/" + file_name) as reader:
+            return reader.read()
+
+
+class ODPSVolumeWriter:
+    def __init__(
+        self,
+        odps_entry: ODPS,
+        volume_name: str,
+        volume_dir: str,
+        schema_name: Optional[str] = None,
+    ):
+        self._odps_entry = odps_entry
+        self._volume = odps_entry.get_volume(volume_name, schema=schema_name)
+        self._volume_dir = volume_dir
+
+    def write_file(self, file_name: str, data: Union[bytes, Iterator[bytes]]):
+        with self._volume.open_writer(self._volume_dir + "/" + file_name) as writer:
+            if not inspect.isgenerator(data):
+                writer.write(data)
+            else:
+                for chunk in data:
+                    writer.write(chunk)

maxframe/learn/contrib/__init__.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from . import pytorch
+from . import graph, llm, pytorch
 
+del graph
+del llm
 del pytorch

maxframe/learn/contrib/graph/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .connected_components import connected_components