maxframe 1.0.0rc3__cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl → 1.1.0__cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl

maxframe/codegen.py

@@ -347,6 +347,7 @@ BUILTIN_ENGINE_SPE = "SPE"
 BUILTIN_ENGINE_MCSQL = "MCSQL"
 
 FAST_RANGE_INDEX_ENABLED = "codegen.fast_range_index_enabled"
+ROW_NUMBER_WINDOW_INDEX_ENABLED = "codegen.row_number_window_index_enabled"
 
 
 class BigDagCodeGenerator(metaclass=abc.ABCMeta):

maxframe/config/config.py

@@ -343,6 +343,9 @@ default_options.register_option("sql.enable_mcqa", True, validator=is_bool, remo
 default_options.register_option(
     "sql.generate_comments", True, validator=is_bool, remote=True
 )
+default_options.register_option(
+    "sql.auto_use_common_image", True, validator=is_bool, remote=True
+)
 default_options.register_option("sql.settings", {}, validator=is_dict, remote=True)
 
 default_options.register_option("is_production", False, validator=is_bool, remote=True)
@@ -371,13 +374,25 @@ default_options.register_option(
     validator=is_numeric,
     remote=True,
 )
+default_options.register_option(
+    "session.quota_name", None, validator=is_null | is_string, remote=True
+)
+default_options.register_option(
+    "session.enable_schema", None, validator=is_null | is_bool, remote=True
+)
+default_options.register_option(
+    "session.enable_high_availability", None, validator=is_null | is_bool, remote=True
+)
+default_options.register_option(
+    "session.default_schema", None, validator=is_null | is_string, remote=True
+)
 default_options.register_option(
     "session.upload_batch_size",
     _DEFAULT_UPLOAD_BATCH_SIZE,
     validator=is_integer,
 )
 default_options.register_option(
-    "session.table_lifecycle", None, validator=is_null | is_integer
+    "session.table_lifecycle", None, validator=is_null | is_integer, remote=True
 )
 default_options.register_option(
     "session.temp_table_lifecycle",

maxframe/conftest.py

@@ -14,10 +14,11 @@
 
 import faulthandler
 import os
-from configparser import ConfigParser, NoOptionError
+from configparser import ConfigParser, NoOptionError, NoSectionError
 
 import pytest
 from odps import ODPS
+from odps.accounts import BearerTokenAccount
 
 from .config import options
 
@@ -34,12 +35,23 @@ def test_config():
     return config
 
 
-@pytest.fixture(scope="session", autouse=True)
-def odps_envs(test_config):
-    access_id = test_config.get("odps", "access_id")
-    secret_access_key = test_config.get("odps", "secret_access_key")
-    project = test_config.get("odps", "project")
-    endpoint = test_config.get("odps", "endpoint")
+def _get_odps_env(test_config: ConfigParser, section_name: str) -> ODPS:
+    try:
+        access_id = test_config.get(section_name, "access_id")
+    except NoOptionError:
+        access_id = test_config.get("odps", "access_id")
+    try:
+        secret_access_key = test_config.get(section_name, "secret_access_key")
+    except NoOptionError:
+        secret_access_key = test_config.get("odps", "secret_access_key")
+    try:
+        project = test_config.get(section_name, "project")
+    except NoOptionError:
+        project = test_config.get("odps", "project")
+    try:
+        endpoint = test_config.get(section_name, "endpoint")
+    except NoOptionError:
+        endpoint = test_config.get("odps", "endpoint")
     try:
         tunnel_endpoint = test_config.get("odps", "tunnel_endpoint")
     except NoOptionError:
@@ -55,12 +67,31 @@ def odps_envs(test_config):
         ],
     }
     token = entry.get_project().generate_auth_token(policy, "bearer", 5)
+    return ODPS(
+        account=BearerTokenAccount(token, 5),
+        project=project,
+        endpoint=endpoint,
+        tunnel_endpoint=tunnel_endpoint,
+    )
+
+
+@pytest.fixture(scope="session")
+def odps_with_schema(test_config):
+    try:
+        return _get_odps_env(test_config, "odps_with_schema")
+    except NoSectionError:
+        pytest.skip("Need to specify odps_with_schema section in test.conf")
+
+
+@pytest.fixture(scope="session", autouse=True)
+def odps_envs(test_config):
+    entry = _get_odps_env(test_config, "odps")
 
-    os.environ["ODPS_BEARER_TOKEN"] = token
-    os.environ["ODPS_PROJECT_NAME"] = project
-    os.environ["ODPS_ENDPOINT"] = endpoint
-    if tunnel_endpoint:
-        os.environ["ODPS_TUNNEL_ENDPOINT"] = tunnel_endpoint
+    os.environ["ODPS_BEARER_TOKEN"] = entry.account.token
+    os.environ["ODPS_PROJECT_NAME"] = entry.project
+    os.environ["ODPS_ENDPOINT"] = entry.endpoint
+    if entry.tunnel_endpoint:
+        os.environ["ODPS_TUNNEL_ENDPOINT"] = entry.tunnel_endpoint
 
     try:
         yield
@@ -95,7 +126,14 @@ def oss_config():
         oss_rolearn = config.get("oss", "rolearn")
 
         options.service_role_arn = oss_rolearn
-        options.object_cache_url = f"oss://{oss_endpoint}/{oss_bucket_name}"
+        if "test" in oss_endpoint:
+            oss_svc_endpoint = oss_endpoint
+        else:
+            endpoint_parts = oss_endpoint.split(".", 1)
+            if "-internal" not in endpoint_parts[0]:
+                endpoint_parts[0] += "-internal"
+            oss_svc_endpoint = ".".join(endpoint_parts)
+        options.object_cache_url = f"oss://{oss_svc_endpoint}/{oss_bucket_name}"
 
         config.oss_config = (
             oss_access_id,
@@ -110,7 +148,7 @@ def oss_config():
         config.oss_bucket = oss2.Bucket(auth, oss_endpoint, oss_bucket_name)
         config.oss_rolearn = oss_rolearn
         yield config
-    except (ConfigParser.NoSectionError, ConfigParser.NoOptionError, ImportError):
+    except (NoSectionError, NoOptionError, ImportError):
         return None
     finally:
         options.service_role_arn = old_role_arn

maxframe/core/entity/executable.py

@@ -46,7 +46,7 @@ class DecrefRunner:
                 break
 
             session = session_ref()
-            if session is None:
+            if session is None or session.closed:
                 fut.set_result(None)
                 continue
             try:

maxframe/core/operator/base.py

@@ -86,6 +86,8 @@ class SchedulingHint(Serializable):
     # `gpu` indicates that if the operator should be executed on the GPU.
     gpu = BoolField("gpu", default=None)
     priority = Int32Field("priority", default=None)
+    expect_engine = StringField("expect_engine", default=None)
+    expect_resources = DictField("expect_resources", FieldTypes.string, default=None)
 
     @classproperty
     @lru_cache(1)

maxframe/dataframe/arithmetic/docstring.py

@@ -185,7 +185,6 @@ e    NaN
 dtype: float64
 """
 
-# FIXME: https://github.com/aliyun/alibabacloud-odps-maxframe-client/issues/48
 _flex_comp_doc_FRAME = """
 Get {desc} of dataframe and other, element-wise (binary operator `{op_name}`).
 Among flexible wrappers (`eq`, `ne`, `le`, `lt`, `ge`, `gt`) to comparison
@@ -291,7 +290,7 @@ C   True    False
 
 Compare to a DataFrame of different shape.
 
->>> other = pd.DataFrame({{'revenue': [300, 250, 100, 150]}},
+>>> other = md.DataFrame({{'revenue': [300, 250, 100, 150]}},
 ...                      index=['A', 'B', 'C', 'D'])
 >>> other.execute()
    revenue
@@ -306,6 +305,31 @@ A  False    False
 B  False    False
 C  False     True
 D  False    False
+
+Compare to a MultiIndex by level.
+
+>>> df_multindex = md.DataFrame({{'cost': [250, 150, 100, 150, 300, 220],
+...                              'revenue': [100, 250, 300, 200, 175, 225]}},
+...                             index=[['Q1', 'Q1', 'Q1', 'Q2', 'Q2', 'Q2'],
+...                                    ['A', 'B', 'C', 'A', 'B', 'C']])
+>>> df_multindex.execute()
+      cost  revenue
+Q1 A   250      100
+   B   150      250
+   C   100      300
+Q2 A   150      200
+   B   300      175
+   C   220      225
+
+>>> df.le(df_multindex, level=1).execute()
+       cost  revenue
+Q1 A   True     True
+   B   True     True
+   C   True     True
+Q2 A  False     True
+   B   True    False
+   C   True    False
+
 """
 
 

maxframe/dataframe/arithmetic/equal.py

@@ -51,6 +51,8 @@ dtype: bool
 
 
 @bin_compare_doc("Equal to", equiv="==", series_example=_eq_example)
-def eq(df, other, axis="columns", level=None):
-    op = DataFrameEqual(axis=axis, level=level, lhs=df, rhs=other)
+def eq(df, other, axis="columns", level=None, fill_value=None):
+    op = DataFrameEqual(
+        axis=axis, level=level, lhs=df, rhs=other, fill_value=fill_value
+    )
     return op(df, other)

maxframe/dataframe/arithmetic/greater.py

@@ -52,6 +52,8 @@ dtype: bool
 
 
 @bin_compare_doc("Greater than", equiv=">", series_example=_gt_example)
-def gt(df, other, axis="columns", level=None):
-    op = DataFrameGreater(axis=axis, level=level, lhs=df, rhs=other)
+def gt(df, other, axis="columns", level=None, fill_value=None):
+    op = DataFrameGreater(
+        axis=axis, level=level, lhs=df, rhs=other, fill_value=fill_value
+    )
     return op(df, other)

maxframe/dataframe/arithmetic/greater_equal.py

@@ -52,6 +52,8 @@ dtype: bool
 
 
 @bin_compare_doc("Greater than or equal to", equiv=">=", series_example=_ge_example)
-def ge(df, other, axis="columns", level=None):
-    op = DataFrameGreaterEqual(axis=axis, level=level, lhs=df, rhs=other)
+def ge(df, other, axis="columns", level=None, fill_value=None):
+    op = DataFrameGreaterEqual(
+        axis=axis, level=level, lhs=df, rhs=other, fill_value=fill_value
+    )
     return op(df, other)

maxframe/dataframe/arithmetic/less.py

@@ -52,6 +52,6 @@ dtype: bool
 
 
 @bin_compare_doc("Less than", equiv="<", series_example=_lt_example)
-def lt(df, other, axis="columns", level=None):
-    op = DataFrameLess(axis=axis, level=level, lhs=df, rhs=other)
+def lt(df, other, axis="columns", level=None, fill_value=None):
+    op = DataFrameLess(axis=axis, level=level, lhs=df, rhs=other, fill_value=fill_value)
     return op(df, other)

maxframe/dataframe/arithmetic/less_equal.py

@@ -52,6 +52,8 @@ dtype: bool
 
 
 @bin_compare_doc("Less than or equal to", equiv="<=", series_example=_le_example)
-def le(df, other, axis="columns", level=None):
-    op = DataFrameLessEqual(axis=axis, level=level, lhs=df, rhs=other)
+def le(df, other, axis="columns", level=None, fill_value=None):
+    op = DataFrameLessEqual(
+        axis=axis, level=level, lhs=df, rhs=other, fill_value=fill_value
+    )
     return op(df, other)

maxframe/dataframe/arithmetic/not_equal.py

@@ -51,6 +51,8 @@ dtype: bool
 
 
 @bin_compare_doc("Not equal to", equiv="!=", series_example=_ne_example)
-def ne(df, other, axis="columns", level=None):
-    op = DataFrameNotEqual(axis=axis, level=level, lhs=df, rhs=other)
+def ne(df, other, axis="columns", level=None, fill_value=None):
+    op = DataFrameNotEqual(
+        axis=axis, level=level, lhs=df, rhs=other, fill_value=fill_value
+    )
     return op(df, other)

maxframe/dataframe/arithmetic/tests/test_arithmetic.py

@@ -22,6 +22,7 @@ import pandas as pd
 import pytest
 
 from ....core import OperatorType
+from ....tests.utils import assert_mf_index_dtype
 from ....utils import dataslots
 from ...core import IndexValue
 from ...datasource.dataframe import from_pandas
@@ -164,7 +165,7 @@ def test_without_shuffle(func_name, func_opts):
     pd.testing.assert_index_equal(
         df3.columns_value.to_pandas(), func_opts.func(data1, data2).columns
     )
-    assert isinstance(df3.index_value.value, IndexValue.Int64Index)
+    assert_mf_index_dtype(df3.index_value.value, np.int64)
     pd.testing.assert_index_equal(
         df3.index_value.to_pandas(), pd.Index([], dtype=np.int64)
     )
@@ -176,7 +177,7 @@ def test_without_shuffle(func_name, func_opts):
     pd.testing.assert_index_equal(
         df3.columns_value.to_pandas(), func_opts.func(data1, data2).columns
     )
-    assert isinstance(df3.index_value.value, IndexValue.Int64Index)
+    assert_mf_index_dtype(df3.index_value.value, np.int64)
     pd.testing.assert_index_equal(
         df3.index_value.to_pandas(), pd.Index([], dtype=np.int64)
     )
@@ -370,7 +371,7 @@ def test_with_one_shuffle(func_name, func_opts):
     pd.testing.assert_index_equal(
         df3.columns_value.to_pandas(), func_opts.func(data1, data2).columns
     )
-    assert isinstance(df3.index_value.value, IndexValue.Int64Index)
+    assert_mf_index_dtype(df3.index_value.value, np.int64)
     pd.testing.assert_index_equal(
         df3.index_value.to_pandas(), pd.Index([], dtype=np.int64)
     )
@@ -403,7 +404,7 @@ def test_with_all_shuffle(func_name, func_opts):
     pd.testing.assert_index_equal(
         df3.columns_value.to_pandas(), func_opts.func(data1, data2).columns
     )
-    assert isinstance(df3.index_value.value, IndexValue.Int64Index)
+    assert_mf_index_dtype(df3.index_value.value, np.int64)
     pd.testing.assert_index_equal(
         df3.index_value.to_pandas(), pd.Index([], dtype=np.int64)
     )
@@ -433,7 +434,7 @@ def test_with_all_shuffle(func_name, func_opts):
     pd.testing.assert_index_equal(
         df6.columns_value.to_pandas(), func_opts.func(data4, data5).columns
     )
-    assert isinstance(df6.index_value.value, IndexValue.Int64Index)
+    assert_mf_index_dtype(df6.index_value.value, np.int64)
     pd.testing.assert_index_equal(
         df6.index_value.to_pandas(), pd.Index([], dtype=np.int64)
     )
@@ -468,7 +469,7 @@ def test_without_shuffle_and_with_one_chunk(func_name, func_opts):
     pd.testing.assert_index_equal(
         df3.columns_value.to_pandas(), func_opts.func(data1, data2).columns
     )
-    assert isinstance(df3.index_value.value, IndexValue.Int64Index)
+    assert_mf_index_dtype(df3.index_value.value, np.int64)
     pd.testing.assert_index_equal(
         df3.index_value.to_pandas(), pd.Index([], dtype=np.int64)
     )
@@ -501,7 +502,7 @@ def test_both_one_chunk(func_name, func_opts):
     pd.testing.assert_index_equal(
         df3.columns_value.to_pandas(), func_opts.func(data1, data2).columns
     )
-    assert isinstance(df3.index_value.value, IndexValue.Int64Index)
+    assert_mf_index_dtype(df3.index_value.value, np.int64)
     pd.testing.assert_index_equal(
         df3.index_value.to_pandas(), pd.Index([], dtype=np.int64)
     )
@@ -534,7 +535,7 @@ def test_with_shuffle_and_one_chunk(func_name, func_opts):
     pd.testing.assert_index_equal(
         df3.columns_value.to_pandas(), func_opts.func(data1, data2).columns
     )
-    assert isinstance(df3.index_value.value, IndexValue.Int64Index)
+    assert_mf_index_dtype(df3.index_value.value, np.int64)
     pd.testing.assert_index_equal(
         df3.index_value.to_pandas(), pd.Index([], dtype=np.int64)
     )
@@ -558,7 +559,7 @@ def test_on_same_dataframe(func_name, func_opts):
     pd.testing.assert_index_equal(
         df2.columns_value.to_pandas(), func_opts.func(data, data).columns
     )
-    assert isinstance(df2.index_value.value, IndexValue.Int64Index)
+    assert_mf_index_dtype(df2.index_value.value, np.int64)
     pd.testing.assert_index_equal(
         df2.index_value.to_pandas(), pd.Index([], dtype=np.int64)
     )
@@ -590,19 +591,19 @@ def test_dataframe_and_scalar(func_name, func_opts):
     pd.testing.assert_series_equal(result.dtypes, expected.dtypes)
 
     pd.testing.assert_index_equal(result.columns_value.to_pandas(), data.columns)
-    assert isinstance(result.index_value.value, IndexValue.Int64Index)
+    assert_mf_index_dtype(result.index_value.value, np.int64)
 
     pd.testing.assert_index_equal(result2.columns_value.to_pandas(), data.columns)
-    assert isinstance(result2.index_value.value, IndexValue.Int64Index)
+    assert_mf_index_dtype(result2.index_value.value, np.int64)
 
     pd.testing.assert_index_equal(result3.columns_value.to_pandas(), data.columns)
-    assert isinstance(result3.index_value.value, IndexValue.Int64Index)
+    assert_mf_index_dtype(result3.index_value.value, np.int64)
 
     pd.testing.assert_index_equal(result4.columns_value.to_pandas(), data.columns)
-    assert isinstance(result4.index_value.value, IndexValue.Int64Index)
+    assert_mf_index_dtype(result4.index_value.value, np.int64)
 
     pd.testing.assert_index_equal(result5.columns_value.to_pandas(), data.columns)
-    assert isinstance(result5.index_value.value, IndexValue.Int64Index)
+    assert_mf_index_dtype(result5.index_value.value, np.int64)
 
     if "builtin_function_or_method" not in str(type(func_opts.func)):
         # skip NotImplemented test for comparison function
@@ -679,7 +680,7 @@ def test_abs():
     pd.testing.assert_index_equal(
         df2.columns_value.to_pandas(), df1.columns_value.to_pandas()
     )
-    assert isinstance(df2.index_value.value, IndexValue.Int64Index)
+    assert_mf_index_dtype(df2.index_value.value, np.int64)
     assert df2.shape == (10, 10)
 
 
@@ -697,7 +698,7 @@ def test_not():
     pd.testing.assert_index_equal(
         df2.columns_value.to_pandas(), df1.columns_value.to_pandas()
     )
-    assert isinstance(df2.index_value.value, IndexValue.Int64Index)
+    assert_mf_index_dtype(df2.index_value.value, np.int64)
     assert df2.shape == (10, 10)
 
 

maxframe/dataframe/core.py

@@ -142,6 +142,14 @@ class IndexValue(Serializable):
         _data = NDArrayField("data")
         _dtype = DataTypeField("dtype")
 
+        @property
+        def dtype(self):
+            return getattr(self, "_dtype", None)
+
+        @property
+        def inferred_type(self):
+            return "floating" if self.dtype.kind == "f" else "integer"
+
     class RangeIndex(IndexBase):
         _name = AnyField("name")
         _slice = SliceField("slice")
@@ -243,6 +251,10 @@ class IndexValue(Serializable):
         _data = NDArrayField("data")
         _dtype = DataTypeField("dtype")
 
+        @property
+        def dtype(self):
+            return getattr(self, "_dtype", None)
+
         @property
         def inferred_type(self):
             return "integer"
@@ -254,6 +266,10 @@ class IndexValue(Serializable):
         _data = NDArrayField("data")
         _dtype = DataTypeField("dtype")
 
+        @property
+        def dtype(self):
+            return getattr(self, "_dtype", None)
+
         @property
         def inferred_type(self):
             return "integer"
@@ -265,6 +281,10 @@ class IndexValue(Serializable):
         _data = NDArrayField("data")
         _dtype = DataTypeField("dtype")
 
+        @property
+        def dtype(self):
+            return getattr(self, "_dtype", None)
+
         @property
         def inferred_type(self):
             return "floating"
@@ -1514,8 +1534,7 @@ class BaseDataFrameData(HasShapeTileableData, _ToPandasMixin):
         refresh_index_value(self)
         refresh_dtypes(self)
 
-    def refresh_from_table_meta(self, table_meta: DataFrameTableMeta) -> None:
-        dtypes = table_meta.pd_column_dtypes
+    def refresh_from_dtypes(self, dtypes: pd.Series) -> None:
         self._dtypes = dtypes
         self._columns_value = parse_index(dtypes.index, store_data=True)
         self._dtypes_value = DtypesValue(key=tokenize(dtypes), value=dtypes)
@@ -1523,6 +1542,9 @@ class BaseDataFrameData(HasShapeTileableData, _ToPandasMixin):
         new_shape[-1] = len(dtypes)
         self._shape = tuple(new_shape)
 
+    def refresh_from_table_meta(self, table_meta: DataFrameTableMeta) -> None:
+        self.refresh_from_dtypes(table_meta.pd_column_dtypes)
+
     @property
     def dtypes(self):
         dt = getattr(self, "_dtypes", None)
@@ -1666,6 +1688,8 @@ class DataFrameData(_BatchedFetcher, BaseDataFrameData):
             raise NotImplementedError
 
         corner_data = fetch_corner_data(self, session=self._executed_sessions[-1])
+        if corner_data is None:
+            return
 
         buf = StringIO()
         max_rows = pd.get_option("display.max_rows")