maxframe 2.2.0__cp312-cp312-win_amd64.whl

maxframe/dataframe/statistics/tests/test_statistics.py

@@ -0,0 +1,82 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import numpy as np
+import pandas as pd
+
+from ....tensor import Tensor
+from ...core import DataFrame, Series
+from ...datasource.dataframe import from_pandas as df_from_pandas
+from ...datasource.series import from_pandas as series_from_pandas
+
+
+def test_series_quantile():
+    raw = pd.Series(np.random.rand(10))
+    s = series_from_pandas(raw, chunk_size=3)
+
+    r = s.quantile()
+    assert isinstance(r, Tensor)
+
+    s = series_from_pandas(raw, chunk_size=3)
+
+    r = s.quantile([0.3, 0.7])
+    assert isinstance(r, Series)
+    assert r.shape == (2,)
+    pd.testing.assert_index_equal(r.index_value.to_pandas(), pd.Index([0.3, 0.7]))
+
+
+def test_dataframe_quantile():
+    raw = pd.DataFrame(
+        {
+            "a": np.random.rand(10),
+            "b": np.random.randint(1000, size=10),
+            "c": [np.random.bytes(5) for _ in range(10)],
+        }
+    )
+    s = df_from_pandas(raw, chunk_size=7)
+
+    # q = 0.3, axis = 0
+    r = s.quantile(0.3)
+    e = raw.quantile(0.3, numeric_only=True)
+    assert isinstance(r, Series)
+    assert r.shape == (2,)
+    assert r.dtype == e.dtype
+    pd.testing.assert_index_equal(r.index_value.to_pandas(), e.index)
+
+    # q = 0.3, axis = 1
+    r = s.quantile(0.3, axis=1)
+    e = raw.quantile(0.3, numeric_only=True, axis=1)
+    assert isinstance(r, Series)
+    assert r.shape == e.shape
+    assert r.dtype == e.dtype
+    pd.testing.assert_index_equal(r.index_value.to_pandas(), e.index)
+
+    # q = [0.3, 0.7], axis = 0
+    r = s.quantile([0.3, 0.7])
+    e = raw.quantile([0.3, 0.7], numeric_only=True)
+    assert isinstance(r, DataFrame)
+    assert r.shape == e.shape
+    pd.testing.assert_series_equal(r.dtypes, e.dtypes)
+    pd.testing.assert_index_equal(r.index_value.to_pandas(), e.index)
+    pd.testing.assert_index_equal(r.columns_value.to_pandas(), e.columns)
+
+    # q = [0.3, 0.7], axis = 1
+    r = s.quantile([0.3, 0.7], axis=1)
+    e = raw.quantile([0.3, 0.7], numeric_only=True, axis=1)
+    assert isinstance(r, DataFrame)
+    assert r.shape == e.shape
+    pd.testing.assert_series_equal(r.dtypes, e.dtypes)
+    pd.testing.assert_index_equal(r.index_value.to_pandas(), e.index)
+    pd.testing.assert_index_equal(r.columns_value.to_pandas(), e.columns)

maxframe/dataframe/tests/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

maxframe/dataframe/tests/test_initializer.py

@@ -0,0 +1,60 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pandas as pd
+import pytest
+
+from ..core import DATAFRAME_TYPE, INDEX_TYPE, SERIES_TYPE
+from ..initializer import DataFrame, Series, read_pandas
+
+
+def test_read_pandas():
+    df_data = pd.DataFrame([["a", 1], ["b", 2]], columns=["a", "b"])
+    assert isinstance(read_pandas(df_data), DATAFRAME_TYPE)
+
+    s_data = pd.Series(["a", "b"])
+    assert isinstance(read_pandas(s_data), SERIES_TYPE)
+
+    idx_data = pd.Index(["a", "b"])
+    assert isinstance(read_pandas(idx_data), INDEX_TYPE)
+
+
+def test_init_dataframe_from_maxframe_series():
+    s = Series([1, 2, 3, 4], index=[1, 2, 3, 4])
+
+    df = DataFrame(s, index=s.index, columns=["col1"])
+
+    assert isinstance(df, DATAFRAME_TYPE)
+    assert df.dtypes.index == ["col1"]
+
+    with pytest.raises(ValueError):
+        DataFrame(s, index=s.index, columns=[])
+
+    with pytest.raises(ValueError):
+        DataFrame(s, index=s.index, columns="col1")
+
+    with pytest.raises(ValueError):
+        DataFrame(s, index=s.index, columns="col2")
+
+
+def test_init_dataframe_from_maxframe_dataframe():
+    df1 = DataFrame({"A": [1, 2, 3, 4], "B": [1, 2, 3, 4]}, index=[1, 2, 3, 4])
+
+    df2 = DataFrame(df1, index=df1.index, columns=["col1", "col2"])
+
+    assert isinstance(df2, DATAFRAME_TYPE)
+    assert list(df2.dtypes.index) == ["col1", "col2"]
+
+    with pytest.raises(ValueError):
+        DataFrame(df1, index=df1.index, columns=["col1", "col2", "col3"])

maxframe/dataframe/tests/test_typing.py

@@ -0,0 +1,104 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+import pandas as pd
+
+from ...core import OutputType
+from ..typing_ import get_function_output_meta
+
+
+def test_dataframe_type_annotation():
+    def func() -> pd.DataFrame[int]:
+        pass
+
+    meta = get_function_output_meta(func)
+    assert meta is not None
+    assert meta.output_type.name == "dataframe"
+    assert len(meta.dtypes) == 1
+    assert meta.dtypes[0] == np.dtype(int)
+
+    def func() -> pd.DataFrame[{"col1": int, "col2": float}]:  # noqa: F821
+        pass
+
+    meta = get_function_output_meta(func)
+    assert meta is not None
+    assert meta.output_type.name == "dataframe"
+    assert len(meta.dtypes) == 2
+    assert meta.dtypes[0] == np.dtype(int)
+    assert meta.dtypes[1] == np.dtype(float)
+
+    def func() -> pd.DataFrame[str, {"col1": int, "col2": float}]:  # noqa: F821
+        pass
+
+    meta = get_function_output_meta(func)
+    assert meta is not None
+    assert meta.output_type.name == "dataframe"
+    assert len(meta.dtypes) == 2
+    assert meta.index_value.value.dtype == np.dtype("O")
+    assert list(meta.dtypes.index) == ["col1", "col2"]
+    assert list(meta.dtypes) == [np.dtype(int), np.dtype(float)]
+
+
+def test_series_type_annotation():
+    def func() -> pd.Series[np.str_]:
+        pass
+
+    meta = get_function_output_meta(func)
+    assert meta is not None
+    assert meta.output_type == OutputType.series
+    assert meta.dtype == np.dtype(np.str_)
+
+    def func() -> pd.Series[("idx_name", str), ("series_name", np.int64)]:  # noqa: F821
+        pass
+
+    meta = get_function_output_meta(func)
+    assert meta is not None
+    assert meta.output_type == OutputType.series
+    assert meta.name == "series_name"
+    assert meta.dtype == np.dtype(np.int64)
+    assert meta.index_value.value._name == "idx_name"
+    assert meta.index_value.value.dtype == np.dtype("O")
+
+
+def test_index_type_annotation():
+    def func() -> pd.Index[np.int64]:
+        pass
+
+    meta = get_function_output_meta(func)
+    assert meta is not None
+    assert meta.output_type == OutputType.index
+
+    def func() -> pd.Index[[("ix1", str), ("ix2", np.int64)]]:  # noqa: F821
+        pass
+
+    meta = get_function_output_meta(func)
+    assert meta is not None
+    assert meta.output_type == OutputType.index
+    assert meta.index_value.value.names == ["ix1", "ix2"]
+    assert list(meta.index_value.value.dtypes) == [np.dtype("O"), np.dtype("int64")]
+
+
+def test_function_output_meta_corner_cases():
+    def func():
+        pass
+
+    assert get_function_output_meta(func) is None
+    assert get_function_output_meta("non-func-obj") is None
+
+    def func() -> int:
+        pass
+
+    meta = get_function_output_meta(func)
+    assert meta.dtype == np.dtype("int64")

maxframe/dataframe/tests/test_utils.py

@@ -0,0 +1,165 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+import pandas as pd
+import pyarrow as pa
+import pytest
+
+from ...config import option_context
+from ...core.operator import Operator
+from ...udf import (
+    MarkedFunction,
+    with_python_requirements,
+    with_resources,
+    with_running_options,
+)
+from ...utils import ARROW_DTYPE_NOT_SUPPORTED
+from ..utils import _generate_value, copy_func_scheduling_hints, pack_func_args
+
+try:
+    from pandas import ArrowDtype
+except:
+    ArrowDtype = None
+
+
+@pytest.fixture
+def df1():
+    return pd.DataFrame([[1, 2], [3, 4]], columns=["a", "b"])
+
+
+def test_pack_function(df1):
+    # pack normal function
+    @with_resources("a.zip")
+    def keep(df):
+        return df
+
+    f = pack_func_args(df1, keep)
+    assert f(df1).equals(df1)
+    assert isinstance(f, MarkedFunction)
+    assert f.resources == ["a.zip"]
+
+    # pack with args
+    @with_python_requirements("numpy")
+    def add(a, b):
+        return a + b
+
+    f = pack_func_args(df1, add, 1)
+    assert f(df1).equals(df1 + 1)
+    assert isinstance(f, MarkedFunction)
+    assert f.pythonpacks[0].requirements == ("numpy",)
+
+    f = pack_func_args(df1, np.sum)
+    assert f(df1).equals(np.sum(df1))
+
+    @with_resources("a.txt")
+    @with_python_requirements("pandas")
+    def times_add(df, param, times):
+        return df * times + param
+
+    f = pack_func_args(df1, times_add, 5, 6)
+    assert f(df1).equals(df1 * 6 + 5)
+    assert isinstance(f, MarkedFunction)
+    assert f.resources == ["a.txt"]
+    assert f.pythonpacks[0].requirements == ("pandas",)
+
+
+@pytest.mark.skipif(ARROW_DTYPE_NOT_SUPPORTED, reason="Arrow Dtype is not supported")
+@pytest.mark.parametrize(
+    "dtype, fill_value, expected",
+    [
+        (
+            ArrowDtype(pa.list_(pa.string())) if ArrowDtype else None,
+            1,
+            ["1"],
+        ),
+        (pa.list_(pa.string()), 1, ["1"]),
+        (
+            ArrowDtype(pa.map_(pa.int32(), pa.string())) if ArrowDtype else None,
+            1,
+            [(np.int32(1), "1")],
+        ),
+        (pa.map_(pa.int32(), pa.string()), 1, [(np.int32(1), "1")]),
+        (
+            ArrowDtype(
+                pa.struct([pa.field("a", pa.int32()), pa.field("b", pa.string())])
+            )
+            if ArrowDtype
+            else None,
+            1,
+            {"a": np.int32(1), "b": "1"},
+        ),
+        (
+            pa.struct([pa.field("a", pa.int32()), pa.field("b", pa.string())]),
+            1,
+            {"a": np.int32(1), "b": "1"},
+        ),
+        (pa.int32(), 1, np.int32(1)),
+        (np.datetime64, "2023-01-01", pd.Timestamp("2023-01-01")),
+        (np.timedelta64, "1D", pd.Timedelta("1D")),
+        (
+            pd.CategoricalDtype(categories=["a", "b", "c"]),
+            "a",
+            pd.CategoricalDtype(["a"]),
+        ),
+        (np.object_, 1, "1"),
+        (np.int32, 1, 1),
+    ],
+)
+def test_generate_value(dtype, fill_value, expected):
+    result = _generate_value(dtype, fill_value)
+    assert result == expected
+
+
+def test_copy_func_scheduling_hints():
+    # Test with a regular function (no scheduling hints)
+    with option_context() as options:
+        options.function.default_running_options = {}  # No default options
+
+        def regular_func(x):
+            return x + 1
+
+        op1 = Operator()
+        copy_func_scheduling_hints(regular_func, op1)
+        # Should not set any attributes since regular function has no hints
+        assert not hasattr(op1, "expect_engine") or op1.expect_engine is None
+        assert not hasattr(op1, "expect_resources") or op1.expect_resources is None
+        assert not hasattr(op1, "gpu") or op1.gpu is None
+
+    # Test with MarkedFunction with scheduling hints
+
+    @with_running_options(engine="DPE", cpu=4, memory="8GiB")
+    def marked_func(x):
+        return x + 1
+
+    op2 = Operator()
+    copy_func_scheduling_hints(marked_func, op2)
+    assert op2.expect_engine == "DPE"
+    # The expect_resources will include default values for gpu and gu_quota
+    expected_resources = {"cpu": 4, "memory": "8GiB", "gpu": 0, "gu_quota": None}
+    assert op2.expect_resources == expected_resources
+
+    # Test with MarkedFunction with GPU
+    @with_running_options(gu=2)
+    def gpu_func(x):
+        return x + 1
+
+    op3 = Operator()
+    copy_func_scheduling_hints(gpu_func, op3)
+    assert op3.gpu is True
+    # The expect_resources will include the gu value and default values
+    # System has default options: {'cpu': 1, 'memory': '4GiB', 'gpu': 0}
+    # The with_running_options decorator will override the gpu value with the gu value
+    expected_resources = {"gpu": 2, "gu_quota": None, "cpu": 1, "memory": "4GiB"}
+    assert op3.expect_resources == expected_resources

maxframe/dataframe/tseries/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

maxframe/dataframe/tseries/tests/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

maxframe/dataframe/tseries/tests/test_tseries.py

@@ -0,0 +1,30 @@
+# Copyright 1999-2025 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pandas as pd
+import pytest
+
+from .... import dataframe as md
+
+
+def test_to_datetime():
+    wrong_args = [pd.DataFrame({"a": [1, 2]}), {"a": [1, 2]}]
+
+    for arg in wrong_args:
+        with pytest.raises(ValueError) as cm:
+            md.to_datetime(arg)
+        assert "[year, month, day]" in str(cm.value)
+
+    with pytest.raises(TypeError):
+        md.to_datetime([[1, 2], [3, 4]])