maxframe 1.1.0__cp311-cp311-win_amd64.whl → 1.1.1__cp311-cp311-win_amd64.whl

maxframe/config/__init__.py

@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .config import AttributeDict, option_context, options
+from .config import AttributeDict, option_context, options, update_wlm_quota_settings

maxframe/config/config.py

@@ -28,6 +28,8 @@ except ImportError:
 
     available_timezones = lambda: all_timezones
 
+import logging
+
 from ..utils import get_python_tag
 from .validators import (
     ValidatorType,
@@ -43,6 +45,8 @@ from .validators import (
     is_valid_cache_path,
 )
 
+logger = logging.getLogger(__name__)
+
 _DEFAULT_REDIRECT_WARN = "Option {source} has been replaced by {target} and might be removed in a future release."
 _DEFAULT_MAX_ALIVE_SECONDS = 3 * 24 * 3600
 _DEFAULT_MAX_IDLE_SECONDS = 3600
@@ -499,3 +503,22 @@ class OptionsProxy:
 
 
 options = OptionsProxy()
+
+
+def update_wlm_quota_settings(session_id: str, engine_settings: Dict[str, Any]):
+    engine_quota = engine_settings.get("odps.task.wlm.quota", None)
+    session_quota = options.session.quota_name or None
+    if engine_quota != session_quota and engine_quota:
+        logger.warning(
+            "[Session=%s] Session quota (%s) is different to SubDag engine quota (%s)",
+            session_id,
+            session_quota,
+            engine_quota,
+        )
+        # TODO(renxiang): overwrite or not overwrite
+        return
+
+    if session_quota:
+        engine_settings["odps.task.wlm.quota"] = session_quota
+    elif "odps.task.wlm.quota" in engine_settings:
+        engine_settings.pop("odps.task.wlm.quota")

maxframe/config/tests/test_config.py

@@ -18,7 +18,14 @@ import threading
 
 import pytest
 
-from ..config import Config, is_integer, is_string, option_context, options
+from ..config import (
+    Config,
+    is_integer,
+    is_string,
+    option_context,
+    options,
+    update_wlm_quota_settings,
+)
 
 
 def test_config_context():
@@ -101,3 +108,15 @@ def test_config_copy():
 
     target_cfg.update(src_cfg_dict)
     assert target_cfg.a.b.c == 1
+
+
+def test_update_wlm_quota_settings():
+    with option_context({}):
+        options.session.quota_name = "quota1"
+        engine_settings = {}
+        update_wlm_quota_settings("session_id", engine_settings)
+        assert engine_settings["odps.task.wlm.quota"] == "quota1"
+        options.session.quota_name = None
+        update_wlm_quota_settings("session_id", engine_settings)
+        # TODO(renxiang): overwrite or not overwrite
+        assert "odps.task.wlm.quota" in engine_settings

maxframe/conftest.py

@@ -40,10 +40,14 @@ def _get_odps_env(test_config: ConfigParser, section_name: str) -> ODPS:
         access_id = test_config.get(section_name, "access_id")
     except NoOptionError:
         access_id = test_config.get("odps", "access_id")
+    if not access_id:
+        access_id = os.getenv("ACCESS_ID")
     try:
         secret_access_key = test_config.get(section_name, "secret_access_key")
     except NoOptionError:
         secret_access_key = test_config.get("odps", "secret_access_key")
+    if not secret_access_key:
+        secret_access_key = os.getenv("SECRET_ACCESS_KEY")
     try:
         project = test_config.get(section_name, "project")
     except NoOptionError:
@@ -119,8 +123,10 @@ def oss_config():
     old_cache_url = options.object_cache_url
 
     try:
-        oss_access_id = config.get("oss", "access_id")
-        oss_secret_access_key = config.get("oss", "secret_access_key")
+        oss_access_id = config.get("oss", "access_id") or os.getenv("ACCESS_ID")
+        oss_secret_access_key = config.get("oss", "secret_access_key") or os.getenv(
+            "SECRET_ACCESS_KEY"
+        )
         oss_bucket_name = config.get("oss", "bucket_name")
         oss_endpoint = config.get("oss", "endpoint")
         oss_rolearn = config.get("oss", "rolearn")

maxframe/dataframe/datasource/read_odps_query.py

@@ -37,6 +37,7 @@ from ...serialization.serializables import (
     SeriesField,
     StringField,
 )
+from ...utils import is_empty
 from ..utils import parse_index
 from .core import ColumnPruneSupportedDataSourceMixin, IncrementalIndexDatasource
 
@@ -250,7 +251,7 @@ class DataFrameReadODPSQuery(
         self.columns = columns
 
     def __call__(self, chunk_bytes=None, chunk_size=None):
-        if not self.index_columns:
+        if is_empty(self.index_columns):
             index_value = parse_index(pd.RangeIndex(0))
         elif len(self.index_columns) == 1:
             index_value = parse_index(

maxframe/dataframe/datasource/read_odps_table.py

@@ -34,6 +34,7 @@ from ...serialization.serializables import (
     SeriesField,
     StringField,
 )
+from ...utils import is_empty
 from ..core import DataFrame  # noqa: F401
 from ..utils import parse_index
 from .core import ColumnPruneSupportedDataSourceMixin, IncrementalIndexDatasource
@@ -76,7 +77,7 @@ class DataFrameReadODPSTable(
         self.columns = columns
 
     def __call__(self, shape, chunk_bytes=None, chunk_size=None):
-        if not self.index_columns:
+        if is_empty(self.index_columns):
             if np.isnan(shape[0]):
                 index_value = parse_index(pd.RangeIndex(0))
             else:
@@ -238,7 +239,8 @@ def read_odps_table(
         partitions = [partitions]
 
     append_partitions = append_partitions or any(
-        pt.name in (columns or ()) for pt in (table.table_schema.partitions or ())
+        pt.name in (columns if not is_empty(columns) else ())
+        for pt in (table.table_schema.partitions or ())
     )
     op = DataFrameReadODPSTable(
         table_name=table.full_table_name,

maxframe/io/odpsio/__init__.py

@@ -19,5 +19,5 @@ from .schema import (
     odps_schema_to_pandas_dtypes,
     pandas_to_odps_schema,
 )
-from .tableio import HaloTableIO, ODPSTableIO
+from .tableio import HaloTableIO, ODPSTableIO, TunnelTableIO
 from .volumeio import ODPSVolumeReader, ODPSVolumeWriter

maxframe/io/odpsio/arrow.py

@@ -14,10 +14,12 @@
 
 from typing import Any, Tuple, Union
 
+import numpy as np
 import pandas as pd
 import pyarrow as pa
 
 from ...core import OutputType
+from ...lib.version import parse as parse_version
 from ...protocol import DataFrameTableMeta
 from ...tensor.core import TENSOR_TYPE
 from ...typing_ import ArrowTableType, PandasObjectTypes
@@ -109,7 +111,26 @@ def pandas_to_arrow(
             df = pd.DataFrame([[df]], columns=names)
     else:  # this could never happen  # pragma: no cover
         raise ValueError(f"Does not support meta type {table_meta.type!r}")
-    pa_table = pa.Table.from_pandas(df, nthreads=nthreads, preserve_index=False)
+
+    try:
+        pa_table = pa.Table.from_pandas(df, nthreads=nthreads, preserve_index=False)
+    except pa.ArrowTypeError as ex:  # pragma: no cover
+        late_np_version = parse_version(np.__version__) >= parse_version("1.20")
+        early_pa_version = parse_version(pa.__version__) <= parse_version("4.0")
+        if (
+            late_np_version
+            and early_pa_version
+            and "Did not pass numpy.dtype object" in str(ex)
+        ):
+            raise TypeError(
+                "Potential dependency conflict. Try update to pyarrow>4.0 "
+                "or downgrade to numpy<1.20. Details can be seen at "
+                "https://github.com/numpy/numpy/issues/17913. "
+                f"Raw error message: {ex!r}"
+            ).with_traceback(ex.__traceback__) from None
+        else:
+            raise
+
     if table_datetime_cols:
         col_names = pa_table.schema.names
         col_datas = []

maxframe/io/odpsio/tableio.py

@@ -15,6 +15,7 @@
 import os
 import time
 from abc import ABC, abstractmethod
+from collections import OrderedDict
 from contextlib import contextmanager
 from typing import Dict, List, Optional, Union
 
@@ -25,7 +26,7 @@ from odps.apis.storage_api import (
     TableBatchScanResponse,
     TableBatchWriteResponse,
 )
-from odps.tunnel import TableTunnel
+from odps.tunnel import TableDownloadSession, TableDownloadStatus, TableTunnel
 from odps.types import OdpsSchema, PartitionSpec, timestamp_ntz
 from odps.utils import call_with_retry
 
@@ -36,12 +37,13 @@ except ImportError:
 
 from ...config import options
 from ...env import ODPS_STORAGE_API_ENDPOINT
-from ...utils import sync_pyodps_options
+from ...utils import is_empty, sync_pyodps_options
 from .schema import odps_schema_to_arrow_schema
 
 PartitionsType = Union[List[str], str, None]
 
 _DEFAULT_ROW_BATCH_SIZE = 4096
+_DOWNLOAD_ID_CACHE_SIZE = 100
 
 
 class ODPSTableIO(ABC):
@@ -65,7 +67,11 @@ class ODPSTableIO(ABC):
     ) -> OdpsSchema:
         final_cols = []
 
-        columns = columns or [col.name for col in table_schema.simple_columns]
+        columns = (
+            columns
+            if not is_empty(columns)
+            else [col.name for col in table_schema.simple_columns]
+        )
         if partition_columns is True:
             partition_columns = [c.name for c in table_schema.partitions]
         else:
@@ -215,6 +221,46 @@ class TunnelMultiPartitionReader:
 
 
 class TunnelTableIO(ODPSTableIO):
+    _down_session_ids = OrderedDict()
+
+    @classmethod
+    def create_download_sessions(
+        cls,
+        odps_entry: ODPS,
+        full_table_name: str,
+        partitions: List[Optional[str]] = None,
+    ) -> Dict[Optional[str], TableDownloadSession]:
+        table = odps_entry.get_table(full_table_name)
+        tunnel = TableTunnel(odps_entry)
+        parts = (
+            [partitions]
+            if partitions is None or isinstance(partitions, str)
+            else partitions
+        )
+        part_to_session = dict()
+        for part in parts:
+            part_key = (full_table_name, part)
+            down_session = None
+
+            if part_key in cls._down_session_ids:
+                down_id = cls._down_session_ids[part_key]
+                down_session = tunnel.create_download_session(
+                    table, async_mode=True, partition_spec=part, download_id=down_id
+                )
+                if down_session.status != TableDownloadStatus.Normal:
+                    down_session = None
+
+            if down_session is None:
+                down_session = tunnel.create_download_session(
+                    table, async_mode=True, partition_spec=part
+                )
+
+            while len(cls._down_session_ids) >= _DOWNLOAD_ID_CACHE_SIZE:
+                cls._down_session_ids.popitem(False)
+            cls._down_session_ids[part_key] = down_session.id
+            part_to_session[part] = down_session
+        return part_to_session
+
     @contextmanager
     def open_reader(
         self,
@@ -241,21 +287,15 @@ class TunnelTableIO(ODPSTableIO):
             or (reverse_range and start is None)
         ):
             with sync_pyodps_options():
-                table = self._odps.get_table(full_table_name)
-                tunnel = TableTunnel(self._odps)
-                parts = (
-                    [partitions]
-                    if partitions is None or isinstance(partitions, str)
-                    else partitions
+                tunnel_sessions = self.create_download_sessions(
+                    self._odps, full_table_name, partitions
+                )
+                part_to_down_id = {
+                    pt: session.id for (pt, session) in tunnel_sessions.items()
+                }
+                total_records = sum(
+                    session.count for session in tunnel_sessions.values()
                 )
-                part_to_down_id = dict()
-                total_records = 0
-                for part in parts:
-                    down_session = tunnel.create_download_session(
-                        table, async_mode=True, partition_spec=part
-                    )
-                    part_to_down_id[part] = down_session.id
-                    total_records += down_session.count
 
         count = None
         if start is not None or stop is not None:

maxframe/serialization/serializables/core.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 import weakref
-from collections import defaultdict
+from collections import OrderedDict
 from typing import Any, Dict, List, Optional, Tuple, Type
 
 import msgpack
@@ -98,14 +98,18 @@ class SerializableMeta(type):
                 non_primitive_fields.append(v)
 
         # count number of fields for every base class
-        cls_to_primitive_field_count = defaultdict(lambda: 0)
-        cls_to_non_primitive_field_count = defaultdict(lambda: 0)
+        cls_to_primitive_field_count = OrderedDict()
+        cls_to_non_primitive_field_count = OrderedDict()
         for field_name in field_order:
             cls_hash = field_to_cls_hash[field_name]
             if field_name in primitive_field_names:
-                cls_to_primitive_field_count[cls_hash] += 1
+                cls_to_primitive_field_count[cls_hash] = (
+                    cls_to_primitive_field_count.get(cls_hash, 0) + 1
+                )
             else:
-                cls_to_non_primitive_field_count[cls_hash] += 1
+                cls_to_non_primitive_field_count[cls_hash] = (
+                    cls_to_non_primitive_field_count.get(cls_hash, 0) + 1
+                )
 
         slots = set(properties.pop("__slots__", set()))
         slots.update(properties_field_slot_names)
@@ -120,9 +124,11 @@ class SerializableMeta(type):
         properties["_FIELD_ORDER"] = field_order
         properties["_FIELD_TO_NAME_HASH"] = field_to_cls_hash
         properties["_PRIMITIVE_FIELDS"] = primitive_fields
-        properties["_CLS_TO_PRIMITIVE_FIELD_COUNT"] = dict(cls_to_primitive_field_count)
+        properties["_CLS_TO_PRIMITIVE_FIELD_COUNT"] = OrderedDict(
+            cls_to_primitive_field_count
+        )
         properties["_NON_PRIMITIVE_FIELDS"] = non_primitive_fields
-        properties["_CLS_TO_NON_PRIMITIVE_FIELD_COUNT"] = dict(
+        properties["_CLS_TO_NON_PRIMITIVE_FIELD_COUNT"] = OrderedDict(
             cls_to_non_primitive_field_count
         )
         properties["__slots__"] = tuple(slots)
@@ -296,21 +302,51 @@ class SerializableSerializer(Serializer):
             else:
                 field.set(obj, value)
 
+    @classmethod
+    def _prune_server_fields(
+        cls,
+        client_cls_to_field_count: Optional[Dict[int, int]],
+        server_cls_to_field_count: Dict[int, int],
+        server_fields: list,
+    ) -> list:
+        if not client_cls_to_field_count:  # pragma: no cover
+            # todo remove this branch when all versions below v0.1.0b5 is eliminated
+            return server_fields
+        if set(client_cls_to_field_count.keys()) == set(
+            server_cls_to_field_count.keys()
+        ):
+            return server_fields
+        ret_server_fields = []
+        server_pos = 0
+        for cls_hash, count in server_cls_to_field_count.items():
+            if cls_hash in client_cls_to_field_count:
+                ret_server_fields.extend(server_fields[server_pos : server_pos + count])
+            server_pos += count
+        return ret_server_fields
+
     @classmethod
     def _set_field_values(
         cls,
         obj: Serializable,
         values: List[Any],
-        client_cls_to_field_count: Optional[Dict[str, int]],
+        client_cls_to_field_count: Optional[Dict[int, int]],
         is_primitive: bool = True,
     ):
         obj_class = type(obj)
         if is_primitive:
             server_cls_to_field_count = obj_class._CLS_TO_PRIMITIVE_FIELD_COUNT
-            server_fields = obj_class._PRIMITIVE_FIELDS
+            server_fields = cls._prune_server_fields(
+                client_cls_to_field_count,
+                server_cls_to_field_count,
+                obj_class._PRIMITIVE_FIELDS,
+            )
         else:
             server_cls_to_field_count = obj_class._CLS_TO_NON_PRIMITIVE_FIELD_COUNT
-            server_fields = obj_class._NON_PRIMITIVE_FIELDS
+            server_fields = cls._prune_server_fields(
+                client_cls_to_field_count,
+                server_cls_to_field_count,
+                obj_class._NON_PRIMITIVE_FIELDS,
+            )
 
         legacy_to_new_hash = {
             c._LEGACY_NAME_HASH: c._NAME_HASH

maxframe/serialization/serializables/tests/test_serializable.py

@@ -221,7 +221,10 @@ def test_compatible_serializable(set_is_ci):
             _ref_val = ReferenceField("ref_val", "MySimpleSerializable")
             _dict_val = DictField("dict_val")
 
-        class MySubSerializable(MySimpleSerializable):
+        class MyMidSerializable(MySimpleSerializable):
+            _i_bool_val = Int64Field("i_bool_val", default=True)
+
+        class MySubSerializable(MyMidSerializable):
             _m_int_val = Int64Field("m_int_val", default=250)
             _m_str_val = StringField("m_str_val", default="SUB_STR")
 

maxframe/tensor/merge/vstack.py

@@ -48,7 +48,7 @@ def vstack(tup):
 
     Examples
     --------
-    >>> import mars.tensor as mt
+    >>> import maxframe.tensor as mt
 
     >>> a = mt.array([1, 2, 3])
     >>> b = mt.array([2, 3, 4])

maxframe/utils.py

@@ -1127,3 +1127,9 @@ def sync_pyodps_options():
 
 def str_to_bool(s: Optional[str]) -> Optional[bool]:
     return s.lower().strip() in ("true", "1") if s is not None else None
+
+
+def is_empty(val):
+    if isinstance(val, (pd.DataFrame, pd.Series, pd.Index)):
+        return val.empty
+    return not bool(val)

{maxframe-1.1.0.dist-info → maxframe-1.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maxframe
-Version: 1.1.0
+Version: 1.1.1
 Summary: MaxFrame operator-based data analyze framework
 Requires-Dist: numpy<2.0.0,>=1.19.0
 Requires-Dist: pandas>=1.0.0
@@ -11,11 +11,11 @@ Requires-Dist: msgpack>=1.0.0
 Requires-Dist: traitlets>=5.0
 Requires-Dist: cloudpickle<3.0.0,>=1.5.0
 Requires-Dist: pyyaml>=5.1
+Requires-Dist: pickle5; python_version < "3.8"
 Requires-Dist: tornado>=6.0
 Requires-Dist: defusedxml>=0.5.0
 Requires-Dist: tqdm>=4.1.0
-Requires-Dist: importlib-metadata>=1.4
-Requires-Dist: pickle5; python_version < "3.8"
+Requires-Dist: importlib_metadata>=1.4
 Provides-Extra: dev
 Requires-Dist: black>=22.3.0; extra == "dev"
 Requires-Dist: flake8>=5.0.4; extra == "dev"

{maxframe-1.1.0.dist-info → maxframe-1.1.1.dist-info}/RECORD

@@ -1,9 +1,9 @@
 maxframe/__init__.py,sha256=RujkARDvD6mhFpR330UQ0UfogvIdae5EjR1euFpTiYA,1036
-maxframe/_utils.cp311-win_amd64.pyd,sha256=LNCWqE3sHoxLD8a8dt0wwB5ZJFaY_N_W4MOnd6SL5g0,306176
+maxframe/_utils.cp311-win_amd64.pyd,sha256=ljsG5RJgNgnl-XMSSbQvdG6VIm63KWWcDjlRsQcksV0,306176
 maxframe/_utils.pxd,sha256=_qHN-lCY1FQgDFIrrqA79Ys0SBdonp9kXRMS93xKSYk,1187
 maxframe/_utils.pyx,sha256=_3p6aJEJ6WZYLcNZ6o4DxoxsxqadTlJXFlgDeFPxqUQ,17564
 maxframe/codegen.py,sha256=8CvmD-1elhT9hK5O3jadhq7G-3AVf7Tb1MNOJcGJWVA,18267
-maxframe/conftest.py,sha256=O821qk5aADhwAF2-VMDFuIc7Ym_f-iDXXThapRZ86Tw,6247
+maxframe/conftest.py,sha256=YDTh7eJqk19UhZE73-18oKPUxlT9ke7S-kUui4rfeJc,6489
 maxframe/env.py,sha256=xY4wjMWIJ4qLsFAQ5F-X5CrVR7dDSWiryPXni0YSK5c,1435
 maxframe/errors.py,sha256=nhQVjRbH5EsyLZhyAufvHpMhfDN6eR8vcrv4sjaI7p8,1000
 maxframe/extension.py,sha256=XKgK2b42i-jfnLc0lBPiBMsGA6HMQ4a12mJc09tMAFc,2752
@@ -13,12 +13,12 @@ maxframe/protocol.py,sha256=TM6jnUsNy634YaI-Ed-alX48ebtGSyFJwM8XvmlGPbg,19526
 maxframe/session.py,sha256=wftB7_3CHU9ljdgS9b-Gz6PNiUToVSScwSG-x3dlu3I,37710
 maxframe/typing_.py,sha256=EsgH2QO4VF0TRdjshKyL0Tmgiy3Ow8doTwEuam5nStE,1220
 maxframe/udf.py,sha256=bzzpmXVz7BioiNW2RxmKEad2E4X5mJpEfKYmLjGlLGA,5317
-maxframe/utils.py,sha256=RYAcum5Y1YgmzotPzTrSuqvRONSQlC7XXhbQqkr6XtA,35679
-maxframe/config/__init__.py,sha256=AHo3deaCm1JnbbRX_udboJEDYrYytdvivp9RFxJcumI,671
-maxframe/config/config.py,sha256=PjTqbLPW2e8DxgMuz9vmY1lxcSdTWaSxhCsheMeWomI,16011
+maxframe/utils.py,sha256=jCZkpT6OtjIlTYArmRiT-WDsh7sGQb_cxbyVGXBiK94,35817
+maxframe/config/__init__.py,sha256=2VB189TzomdYUKOdSvyb5trVPOXVYX6bS1DzDv5U5YA,698
+maxframe/config/config.py,sha256=2rqp6JPNrUW6FJAwmAxZal3512e8T3uKLupAldxWaMg,16804
 maxframe/config/validators.py,sha256=vs82UJa4xdY7tvbM7U4vOtXDJHrfdAOTvDkbrFtuGLU,2599
 maxframe/config/tests/__init__.py,sha256=29eM5D4knhYwe3TF42naTuC5b4Ym3VeH4rK8KpdLWNY,609
-maxframe/config/tests/test_config.py,sha256=FWQZ6KBUG_jY1-KaR-GKXl7khhlTbuLlk3uaEV8koM8,2839
+maxframe/config/tests/test_config.py,sha256=oi_7HOVIEDARpNnflDFAC5tkoP3Kk188VXU5Or-ewpI,3405
 maxframe/config/tests/test_validators.py,sha256=kz9Yxiqvl_BJq2CT61JTzfpRyrjm6lFrjqIfqeld_yo,1274
 maxframe/core/__init__.py,sha256=uNccOOfZ1aCxvUsz8ic5kObQkrBgfUv8ExU7OScyW1w,1508
 maxframe/core/base.py,sha256=43qZ_sJFgW857qhT1jqgsjdAsBy9tzCeqeX62CPm4vM,4691
@@ -33,7 +33,7 @@ maxframe/core/entity/utils.py,sha256=454RYVbTMVW_8KnfDqUPec4kz1p98izVTC2OrzhOkao
 maxframe/core/entity/tests/__init__.py,sha256=29eM5D4knhYwe3TF42naTuC5b4Ym3VeH4rK8KpdLWNY,609
 maxframe/core/entity/tests/test_objects.py,sha256=8OAMgphXRVri3UewnCYoXfS3lJ_53yxFmR3rv2pWiMY,1392
 maxframe/core/graph/__init__.py,sha256=IRTkHcVzI57tPXZ3agJJb-atpoD-9Jk4vtcRj3XQXzg,782
-maxframe/core/graph/core.cp311-win_amd64.pyd,sha256=iGGd9Fy7X7yPE4Au0R4h2hCAA6tnEtQp98EaY-ajXaU,251392
+maxframe/core/graph/core.cp311-win_amd64.pyd,sha256=PdlGalLtOQmT7N3ZSkGYw4M367D2FEg6HRfWh8ogPAg,250880
 maxframe/core/graph/core.pyx,sha256=J619C6xfVCF8NQ8oeinhICYSg-w5ecX-dnBI7NB-6n0,16390
 maxframe/core/graph/entity.py,sha256=7DJtHi7Xum_Yp16caiQd2AMZd34PWDPy7kocjWhgl3o,5010
 maxframe/core/graph/builder/__init__.py,sha256=fL_blRc623cJ5CQTiFG1d4OK8M97FcFqGbX9chwbUQs,655
@@ -118,8 +118,8 @@ maxframe/dataframe/datasource/from_records.py,sha256=ygpKOMXZnDdWzGxMxQ4KdGv-tJF
 maxframe/dataframe/datasource/from_tensor.py,sha256=mShHYi0fZcG7ZShFVgIezaphh8tSFqR9-nQMm5YKIhw,15146
 maxframe/dataframe/datasource/index.py,sha256=X_NShW67nYJGxaWp3qOrvyInNkz9L-XHjbApU4fHoes,4518
 maxframe/dataframe/datasource/read_csv.py,sha256=IvQihmpcZIdzSD7ziX92aTAHNyP5WnTgd2cZz_h43sQ,24668
-maxframe/dataframe/datasource/read_odps_query.py,sha256=Uao52Qg9o1hZAORIw5Sfphx9bNhd5BWg4EoXHq_2Bn8,13861
-maxframe/dataframe/datasource/read_odps_table.py,sha256=9cMQZ3pdJbL54xHJE2o884iTrwv6VDRkjfehdOa6SOI,9594
+maxframe/dataframe/datasource/read_odps_query.py,sha256=KRk2dAw_8XbrkYzB8AynVUP1PXTartRSOwfg2zgm1QY,13898
+maxframe/dataframe/datasource/read_odps_table.py,sha256=fcH-ZkaO-Ln8QyvjZQNWCoUiKXk-hhTzaai-uQpHgE8,9667
 maxframe/dataframe/datasource/read_parquet.py,sha256=SZPrWoax2mwMBNvRk_3lkS72pZLe-_X_GwQ1JROBMs4,14952
 maxframe/dataframe/datasource/series.py,sha256=elQVupKETh-hUHI2fTu8TRxBE729Vyrmpjx17XlRV-8,1964
 maxframe/dataframe/datasource/tests/__init__.py,sha256=29eM5D4knhYwe3TF42naTuC5b4Ym3VeH4rK8KpdLWNY,609
@@ -289,10 +289,10 @@ maxframe/io/objects/core.py,sha256=rhEYmS8Ty_cqJfjvTiPU-KaUVnr9DwPSIZEaPMZDH2U,4
 maxframe/io/objects/tensor.py,sha256=NQaYIItNYaeM0EcJ6TFejNWSSpXKSJjb6eI_ZKzXf6w,2749
 maxframe/io/objects/tests/__init__.py,sha256=29eM5D4knhYwe3TF42naTuC5b4Ym3VeH4rK8KpdLWNY,609
 maxframe/io/objects/tests/test_object_io.py,sha256=9SSR8XCDoFIuAj6kafRQ9fKW68pvg8pT2gmqFY8BIfk,3899
-maxframe/io/odpsio/__init__.py,sha256=xJbh1eHNI0D3xJG22VM3AHwwiwMIPldtGEJjildPsxc,925
-maxframe/io/odpsio/arrow.py,sha256=Q83DRXCsg1HEuv821Y39i3bdddGHPh-ToKj9ZJ2Tx9g,5231
+maxframe/io/odpsio/__init__.py,sha256=7n5ZtsGR4f7B8yA7YPQodd1g6ecvjg5P4baomogUgsA,940
+maxframe/io/odpsio/arrow.py,sha256=PRZb1hMDcN1yRT8WmZNFI-YCwkj3kxyNryTwLgNsUyE,6073
 maxframe/io/odpsio/schema.py,sha256=HRlz_RqiN86pW6kV2jh-fcrrwRPLfkZtWmUHTTVLVrs,13367
-maxframe/io/odpsio/tableio.py,sha256=N-YW_ynkrpHKXzrc4sE0Yj26kqqrRhtX-70az-t4EH4,21679
+maxframe/io/odpsio/tableio.py,sha256=vDRJW5rELwSC2aB3L4LbCNMFtBE27ZG7SxpK97m6V5U,23116
 maxframe/io/odpsio/volumeio.py,sha256=L-lp6vVGSASmpnsVtA1IC-j5iEOLXEUzXnr82bByveE,3035
 maxframe/io/odpsio/tests/__init__.py,sha256=29eM5D4knhYwe3TF42naTuC5b4Ym3VeH4rK8KpdLWNY,609
 maxframe/io/odpsio/tests/test_arrow.py,sha256=yeDWFzsm2IMS-k6jimlQ7uim5T6WW1Anuy8didaf4cs,3194
@@ -332,7 +332,7 @@ maxframe/learn/utils/core.py,sha256=WNDISxPFsWmjkwHwEvjVGCkAOkIftqzEQFPA_KWr7FY,
 maxframe/lib/__init__.py,sha256=_PB28W40qku6YiT8fJYqdmEdRMQfelOwGeksCOZJfCc,657
 maxframe/lib/compression.py,sha256=QQpNK79iUC9zck74I0HKMhapSRnLBXtTRyS91taEVIc,1497
 maxframe/lib/functools_compat.py,sha256=2LTrkSw5i-z5E9XCtZzfg9-0vPrYxicKvDjnnNrAL1Q,2697
-maxframe/lib/mmh3.cp311-win_amd64.pyd,sha256=mM09OEoxm-yi5jq_zhKlih5Cq23SR2r0Cgslru0Xtt4,17920
+maxframe/lib/mmh3.cp311-win_amd64.pyd,sha256=2JHSE12UEQGNgEg1c948G2p2w2y6qhTF4efSKeGcnbg,17920
 maxframe/lib/mmh3.pyi,sha256=pkzO6SUUukCty3X-WFsuokWBtIPWyxEa06ypF9liruI,1537
 maxframe/lib/version.py,sha256=VOVZu3KHS53YUsb_vQsT7AyHwcCWAgc-3bBqV5ANcbQ,18941
 maxframe/lib/wrapped_pickle.py,sha256=Akx-qhMMJJ6VVzQYrcVO_umFjx0IR-Yzb1XqyOS1Mio,3976
@@ -386,7 +386,7 @@ maxframe/remote/core.py,sha256=ELCUvg-_ZdTax_exFqm6-XWUXukT5qalKYBeDB-mV0k,6733
 maxframe/remote/run_script.py,sha256=k93-vaFLUanWoBRai4-78DX_SLeZ8_rbbxcCtOIXZO8,3677
 maxframe/serialization/__init__.py,sha256=UmDfKNohvk3StL-87QYmQTcQ2g6dRHSk49VFQS73VB0,963
 maxframe/serialization/arrow.py,sha256=OMeDjLcPgagqzokG7g3Vhwm6Xw1j-Kph1V2QsIwi6dw,3513
-maxframe/serialization/core.cp311-win_amd64.pyd,sha256=8a9UgDfEDRnkbyXdw1uWL34o9L_RAla1_5wVudldyzU,423424
+maxframe/serialization/core.cp311-win_amd64.pyd,sha256=FOqTSPY8KFQKr0IvwoB7J9fzAi07nIKcHLoXCrjfWAw,423424
 maxframe/serialization/core.pxd,sha256=KioRiFhr5DTuqXnS2imJ3djWfSv2IAmhnz-kAFPgU6A,1548
 maxframe/serialization/core.pyi,sha256=ELDG44v8O7A7RfURExMfVEJENuaEYHTwgJ-vzlH2Vh4,2206
 maxframe/serialization/core.pyx,sha256=TKEAyaJJAUV20vR_uI93jA6B8mhTt7uoOtMdw0vL5q4,36686
@@ -396,12 +396,12 @@ maxframe/serialization/numpy.py,sha256=ENrFKl24mtYyO1vZRLwHvMD0r4z_UI7J2-yNlmfWS
 maxframe/serialization/pandas.py,sha256=PMp999KR9xnTFSgRTgepbsD7nGfObNwaYeK4UGJ_T4U,8725
 maxframe/serialization/scipy.py,sha256=fGwQ5ZreymrMT8g7TneATfFdKFF7YPNZQqgWgMa3J8M,2498
 maxframe/serialization/serializables/__init__.py,sha256=rlQhIaSAVzz4KYkc5shEHFZDPd6WDMPkxalU76yjJ3M,1406
-maxframe/serialization/serializables/core.py,sha256=LARyWxWwPK-eknPHrMGauPG1GJpfjl-mKI50bQSWuz4,16583
+maxframe/serialization/serializables/core.py,sha256=1YjMfOi3kOCcOaiax1v8ScfXn7gwol70JwXcouaMj5w,17972
 maxframe/serialization/serializables/field.py,sha256=DVott3HAbne4UvN-heSFS9gSl0wCxV5RssS738FCjzk,16639
 maxframe/serialization/serializables/field_type.py,sha256=NtzY4Zi87futV3Er3tD1qK8UpI7MXOC09h2Uipq9RRA,15525
 maxframe/serialization/serializables/tests/__init__.py,sha256=29eM5D4knhYwe3TF42naTuC5b4Ym3VeH4rK8KpdLWNY,609
 maxframe/serialization/serializables/tests/test_field_type.py,sha256=uG87-bdG8xGmjrubEHCww1ZKmRupSvnNKnZoV2SnwYM,4502
-maxframe/serialization/serializables/tests/test_serializable.py,sha256=ZzQcsL82r6l0Ny80MdBXJGohoZ2UcEgdA_5bxEjHX6E,10447
+maxframe/serialization/serializables/tests/test_serializable.py,sha256=CfzLa635DR-9xMdNaGhWohzoICFs3pyKCljPYDQEkr4,10568
 maxframe/serialization/tests/__init__.py,sha256=29eM5D4knhYwe3TF42naTuC5b4Ym3VeH4rK8KpdLWNY,609
 maxframe/serialization/tests/test_serial.py,sha256=D1cCXylizxdQm6tBwVSVDs7v8ZZJokv7jHN_-nrqf7U,12956
 maxframe/tensor/__init__.py,sha256=3XN24f4B0eBroWt7XW33_vJCXlbcJj83zgnmAJpz1dQ,4910
@@ -544,7 +544,7 @@ maxframe/tensor/indexing/tests/test_indexing.py,sha256=uatTFiNkYnyqibYa2-NleUWyF
 maxframe/tensor/merge/__init__.py,sha256=mzXzvbNObPYmvWY3PCu3yFvV82Tqqaxav5DX_Iv2UN8,703
 maxframe/tensor/merge/concatenate.py,sha256=RRyP-fqUGfMeP3Zo2wgfZcnF-u8dCCIOpr_0jI5Fft8,3314
 maxframe/tensor/merge/stack.py,sha256=lZJs8M1JYBdYXnalj6-6ltdJKKslZ1Z-ydkzvGlzzkY,4277
-maxframe/tensor/merge/vstack.py,sha256=RDlTE2A51b1ku7ep1J0p-bq8EQHUkiXLa0rVHNWOLBI,2338
+maxframe/tensor/merge/vstack.py,sha256=T9lGknzxA62FQJUEoPieZ7iFjCcvgkqfR8zFD9ZMG2Y,2342
 maxframe/tensor/merge/tests/__init__.py,sha256=29eM5D4knhYwe3TF42naTuC5b4Ym3VeH4rK8KpdLWNY,609
 maxframe/tensor/merge/tests/test_merge.py,sha256=tI4xeJPpME68buemIzTga9mTQiMoqFc9v4__Hi40kzg,2291
 maxframe/tensor/misc/__init__.py,sha256=C6VvY-dnO0_8Z1ArbzhT0I0ClFNT2LGfSgPJdICUexo,1185
@@ -656,20 +656,20 @@ maxframe/tests/test_utils.py,sha256=5Z2cym1tlpnF73f4I3WC1eBocsbNEDGzDErJL_fKWx8,
 maxframe/tests/utils.py,sha256=lrUZul-tUjc-S7HtJDZs5PcrsA4Xecd96xZmNQ1WX-A,5532
 maxframe_client/__init__.py,sha256=hIVOnxj6AoN2zIMxQCzRb10k0LSoYS_DrQevXO9KPBg,705
 maxframe_client/conftest.py,sha256=UWWMYjmohHL13hLl4adb0gZPLRdBVOYVvsFo6VZruI0,658
-maxframe_client/fetcher.py,sha256=hfzgxDolyI7JtykCpr5Xq3lAQCJGbL2hXaRLXlMlCf0,9477
+maxframe_client/fetcher.py,sha256=cIt1R2k0YV5IKxDQ276x5Dzwk3vFKFyDH3nzRzb6SH0,9445
 maxframe_client/clients/__init__.py,sha256=29eM5D4knhYwe3TF42naTuC5b4Ym3VeH4rK8KpdLWNY,609
 maxframe_client/clients/framedriver.py,sha256=GdF_VGr2n4QZTbHy-p_IAaUrHLD1t7DYA-g3BtnylAA,4678
 maxframe_client/session/__init__.py,sha256=9zFCd3zkSADESAFc4SPoQ2nkvRwsIhhpNNO2TtSaWbU,854
 maxframe_client/session/consts.py,sha256=USDwwz79zFjhlp2MGz5Th6cCuZaYRLAo491tUanZdDs,1430
 maxframe_client/session/graph.py,sha256=CEavpl_zmz_nZ2TXI4moRxpdYsMZMokV63889qI6l_4,4501
-maxframe_client/session/odps.py,sha256=4Fez9gIKdw_HCHRWujmWUe1ZalSsabClVQnp5mzuJto,24803
-maxframe_client/session/task.py,sha256=AYd8DyvmAtYqW_O0_eyMC076O1VG6LvDP7MbpqHzlEg,12175
+maxframe_client/session/odps.py,sha256=HlJ25wt0DLzMrhQVt-izEfrfSEejGMD8D08_x4Xcdto,25586
+maxframe_client/session/task.py,sha256=6uljv5T-VEGKoWDA-E7iSaQMkimQON2vBaunTNLl8HQ,12347
 maxframe_client/session/tests/__init__.py,sha256=29eM5D4knhYwe3TF42naTuC5b4Ym3VeH4rK8KpdLWNY,609
-maxframe_client/session/tests/test_task.py,sha256=kEGse7pEY9pC_Bg2kwFBFILIxIe_84_cPfp-p6MhI7I,3223
+maxframe_client/session/tests/test_task.py,sha256=pzluiWktDx5e7XfJKsasNMAJk83l5RIGbrphwy__yAU,4815
 maxframe_client/tests/__init__.py,sha256=29eM5D4knhYwe3TF42naTuC5b4Ym3VeH4rK8KpdLWNY,609
 maxframe_client/tests/test_fetcher.py,sha256=lYtn4NuntN-_A7rMd10e2y_CEUp7zelTij441NTFP6M,4265
 maxframe_client/tests/test_session.py,sha256=Rn144kIFfJ-ppWNgQ8psLTo0Z_zOk7q3xZCtdiaK_OM,11542
-maxframe-1.1.0.dist-info/METADATA,sha256=iOvh1zxFdLLOm6Q8tlQnTVGtQQoHl6T0KtQxVNf7bXQ,3126
-maxframe-1.1.0.dist-info/WHEEL,sha256=pa6Q1C4C_AVXqyAePbbb0HCsKIpaeBhf3-2iCk_ZSEA,102
-maxframe-1.1.0.dist-info/top_level.txt,sha256=64x-fc2q59c_vXwNUkehyjF1vb8JWqFSdYmUqIFqoTM,31
-maxframe-1.1.0.dist-info/RECORD,,
+maxframe-1.1.1.dist-info/METADATA,sha256=SjFIBOd43-2F8ZPWIvamXD2pDQUA9ciTHsW_FevDxK8,3126
+maxframe-1.1.1.dist-info/WHEEL,sha256=nkBcd8Ko0v5sEcSagm2-x_RVrb8gBSkTa8VFFZ0Mr1o,101
+maxframe-1.1.1.dist-info/top_level.txt,sha256=64x-fc2q59c_vXwNUkehyjF1vb8JWqFSdYmUqIFqoTM,31
+maxframe-1.1.1.dist-info/RECORD,,

{maxframe-1.1.0.dist-info → maxframe-1.1.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.45.0)
+Generator: setuptools (75.6.0)
 Root-Is-Purelib: false
 Tag: cp311-cp311-win_amd64
 

maxframe_client/fetcher.py

@@ -20,7 +20,6 @@ import pandas as pd
 import pyarrow as pa
 from odps import ODPS
 from odps.models import ExternalVolume
-from odps.tunnel import TableTunnel
 
 from maxframe.core import OBJECT_TYPE
 from maxframe.dataframe.core import DATAFRAME_TYPE
@@ -28,6 +27,7 @@ from maxframe.io.objects import get_object_io_handler
 from maxframe.io.odpsio import (
     ODPSTableIO,
     ODPSVolumeReader,
+    TunnelTableIO,
     arrow_to_pandas,
     build_dataframe_table_meta,
     odps_schema_to_pandas_dtypes,
@@ -136,11 +136,10 @@ class ODPSTableFetcher(ToThreadMixin, ResultFetcher):
                     dtypes = odps_schema_to_pandas_dtypes(table.table_schema)
                     tileable.refresh_from_dtypes(dtypes)
 
-                tunnel = TableTunnel(self._odps_entry)
-                total_records = 0
-                for part_spec in part_specs:
-                    session = tunnel.create_download_session(table, part_spec)
-                    total_records += session.count
+                part_sessions = TunnelTableIO.create_download_sessions(
+                    self._odps_entry, info.full_table_name, part_specs
+                )
+                total_records = sum(session.count for session in part_sessions.values())
 
             new_shape_list = list(tileable.shape)
             new_shape_list[0] = total_records

maxframe_client/session/odps.py

@@ -84,10 +84,21 @@ class MaxFrameServiceCaller(metaclass=abc.ABCMeta):
     def get_settings_to_upload(self) -> Dict[str, Any]:
         sql_settings = (odps_options.sql.settings or {}).copy()
         sql_settings.update(options.sql.settings or {})
-
         quota_name = options.session.quota_name or getattr(
             odps_options, "quota_name", None
         )
+        quota_settings = {
+            sql_settings.get("odps.task.wlm.quota", None),
+            options.spe.task.settings.get("odps.task.wlm.quota", None),
+            options.pythonpack.task.settings.get("odps.task.wlm.quota", None),
+            quota_name,
+        }.difference([None])
+        if len(quota_settings) >= 2:
+            raise ValueError(
+                "Quota settings are conflicting: %s" % ", ".join(sorted(quota_settings))
+            )
+        elif len(quota_settings) == 1:
+            quota_name = quota_settings.pop()
         lifecycle = options.session.table_lifecycle or odps_options.lifecycle
         temp_lifecycle = (
             options.session.temp_table_lifecycle or odps_options.temp_lifecycle
@@ -332,6 +343,11 @@ class MaxFrameSession(ToThreadMixin, IsolatedAsyncSession):
             self._last_settings = copy.deepcopy(new_settings)
             return new_settings
 
+        if self._last_settings.get("session.quota_name", None) != new_settings.get(
+            "session.quota_name", None
+        ):
+            raise ValueError("Quota name cannot be changed after sessions are created")
+
         update = dict()
         for k in new_settings.keys():
             old_item = self._last_settings.get(k)

maxframe_client/session/task.py

@@ -126,10 +126,13 @@ class MaxFrameInstanceCaller(MaxFrameServiceCaller):
 
     def _create_maxframe_task(self) -> MaxFrameTask:
         task = MaxFrameTask(name=self._task_name, major_version=self._major_version)
+        mf_settings = self.get_settings_to_upload()
         mf_opts = {
-            "odps.maxframe.settings": json.dumps(self.get_settings_to_upload()),
+            "odps.maxframe.settings": json.dumps(mf_settings),
             "odps.maxframe.output_format": self._output_format,
         }
+        if mf_settings.get("session.quota_name", None):
+            mf_opts["odps.task.wlm.quota"] = mf_settings["session.quota_name"]
         if mf_version:
             mf_opts["odps.maxframe.client_version"] = mf_version
         task.update_settings(mf_opts)

maxframe_client/session/tests/test_task.py

@@ -11,17 +11,20 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 import json
 import os
 
 import mock
+import pytest
 from defusedxml import ElementTree
 from odps import ODPS
 from odps import options as odps_options
 
+from maxframe import options
+from maxframe.config import option_context
+
 from ...session.consts import MAXFRAME_OUTPUT_JSON_FORMAT
-from ...session.task import MaxFrameInstanceCaller, MaxFrameTask
+from ...session.task import MaxFrameInstanceCaller, MaxFrameTask, MaxFrameTaskSession
 
 expected_file_dir = os.path.join(os.path.dirname(__file__), "expected-data")
 
@@ -79,3 +82,33 @@ def test_maxframe_instance_caller_creating_session():
         finally:
             odps_options.priority = old_priority
             odps_options.get_priority = old_get_priority
+
+
+@pytest.mark.asyncio
+async def test_session_quota_flag_valid():
+    def mock_create(self, task: MaxFrameTask, **kwargs):
+        assert task.properties["settings"]
+        task_settings = json.loads(task.properties["settings"])
+        assert task_settings["odps.task.wlm.quota"] == "session_quota"
+
+    with mock.patch.multiple(
+        target="maxframe_client.session.task.MaxFrameInstanceCaller",
+        _wait_instance_task_ready=mock.DEFAULT,
+        get_session=mock.DEFAULT,
+        get_logview_address=mock.DEFAULT,
+    ), mock.patch("odps.models.instances.BaseInstances.create", mock_create):
+        with option_context({"session.quota_name": "session_quota"}):
+            with pytest.raises(ValueError):
+                options.sql.settings["odps.task.wlm.quota"] = "session_quota2"
+                await MaxFrameTaskSession.init(
+                    address="test", odps_entry=ODPS.from_environments()
+                )
+            options.sql.settings["odps.task.wlm.quota"] = "session_quota"
+            mf_task_session = await MaxFrameTaskSession.init(
+                address="test", odps_entry=ODPS.from_environments()
+            )
+            with pytest.raises(ValueError):
+                options.sql.settings["odps.task.wlm.quota"] = "session_quota2"
+                mf_task_session._get_diff_settings()
+            options.sql.settings["odps.task.wlm.quota"] = "session_quota"
+            mf_task_session._get_diff_settings()