maxframe 1.1.0__cp311-cp311-macosx_10_9_x86_64.whl → 1.1.1__cp311-cp311-macosx_10_9_x86_64.whl

maxframe/config/__init__.py

@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .config import AttributeDict, option_context, options
+from .config import AttributeDict, option_context, options, update_wlm_quota_settings

maxframe/config/config.py

@@ -28,6 +28,8 @@ except ImportError:
 
     available_timezones = lambda: all_timezones
 
+import logging
+
 from ..utils import get_python_tag
 from .validators import (
     ValidatorType,
@@ -43,6 +45,8 @@ from .validators import (
     is_valid_cache_path,
 )
 
+logger = logging.getLogger(__name__)
+
 _DEFAULT_REDIRECT_WARN = "Option {source} has been replaced by {target} and might be removed in a future release."
 _DEFAULT_MAX_ALIVE_SECONDS = 3 * 24 * 3600
 _DEFAULT_MAX_IDLE_SECONDS = 3600
@@ -499,3 +503,22 @@ class OptionsProxy:
 
 
 options = OptionsProxy()
+
+
+def update_wlm_quota_settings(session_id: str, engine_settings: Dict[str, Any]):
+    engine_quota = engine_settings.get("odps.task.wlm.quota", None)
+    session_quota = options.session.quota_name or None
+    if engine_quota != session_quota and engine_quota:
+        logger.warning(
+            "[Session=%s] Session quota (%s) is different to SubDag engine quota (%s)",
+            session_id,
+            session_quota,
+            engine_quota,
+        )
+        # TODO(renxiang): overwrite or not overwrite
+        return
+
+    if session_quota:
+        engine_settings["odps.task.wlm.quota"] = session_quota
+    elif "odps.task.wlm.quota" in engine_settings:
+        engine_settings.pop("odps.task.wlm.quota")

maxframe/config/tests/test_config.py

@@ -18,7 +18,14 @@ import threading
 
 import pytest
 
-from ..config import Config, is_integer, is_string, option_context, options
+from ..config import (
+    Config,
+    is_integer,
+    is_string,
+    option_context,
+    options,
+    update_wlm_quota_settings,
+)
 
 
 def test_config_context():
@@ -101,3 +108,15 @@ def test_config_copy():
 
     target_cfg.update(src_cfg_dict)
     assert target_cfg.a.b.c == 1
+
+
+def test_update_wlm_quota_settings():
+    with option_context({}):
+        options.session.quota_name = "quota1"
+        engine_settings = {}
+        update_wlm_quota_settings("session_id", engine_settings)
+        assert engine_settings["odps.task.wlm.quota"] == "quota1"
+        options.session.quota_name = None
+        update_wlm_quota_settings("session_id", engine_settings)
+        # TODO(renxiang): overwrite or not overwrite
+        assert "odps.task.wlm.quota" in engine_settings

maxframe/conftest.py

@@ -40,10 +40,14 @@ def _get_odps_env(test_config: ConfigParser, section_name: str) -> ODPS:
         access_id = test_config.get(section_name, "access_id")
     except NoOptionError:
         access_id = test_config.get("odps", "access_id")
+    if not access_id:
+        access_id = os.getenv("ACCESS_ID")
     try:
         secret_access_key = test_config.get(section_name, "secret_access_key")
     except NoOptionError:
         secret_access_key = test_config.get("odps", "secret_access_key")
+    if not secret_access_key:
+        secret_access_key = os.getenv("SECRET_ACCESS_KEY")
     try:
         project = test_config.get(section_name, "project")
     except NoOptionError:
@@ -119,8 +123,10 @@ def oss_config():
     old_cache_url = options.object_cache_url
 
     try:
-        oss_access_id = config.get("oss", "access_id")
-        oss_secret_access_key = config.get("oss", "secret_access_key")
+        oss_access_id = config.get("oss", "access_id") or os.getenv("ACCESS_ID")
+        oss_secret_access_key = config.get("oss", "secret_access_key") or os.getenv(
+            "SECRET_ACCESS_KEY"
+        )
         oss_bucket_name = config.get("oss", "bucket_name")
         oss_endpoint = config.get("oss", "endpoint")
         oss_rolearn = config.get("oss", "rolearn")

maxframe/dataframe/datasource/read_odps_query.py

@@ -37,6 +37,7 @@ from ...serialization.serializables import (
     SeriesField,
     StringField,
 )
+from ...utils import is_empty
 from ..utils import parse_index
 from .core import ColumnPruneSupportedDataSourceMixin, IncrementalIndexDatasource
 
@@ -250,7 +251,7 @@ class DataFrameReadODPSQuery(
         self.columns = columns
 
     def __call__(self, chunk_bytes=None, chunk_size=None):
-        if not self.index_columns:
+        if is_empty(self.index_columns):
             index_value = parse_index(pd.RangeIndex(0))
         elif len(self.index_columns) == 1:
             index_value = parse_index(

maxframe/dataframe/datasource/read_odps_table.py

@@ -34,6 +34,7 @@ from ...serialization.serializables import (
     SeriesField,
     StringField,
 )
+from ...utils import is_empty
 from ..core import DataFrame  # noqa: F401
 from ..utils import parse_index
 from .core import ColumnPruneSupportedDataSourceMixin, IncrementalIndexDatasource
@@ -76,7 +77,7 @@ class DataFrameReadODPSTable(
         self.columns = columns
 
     def __call__(self, shape, chunk_bytes=None, chunk_size=None):
-        if not self.index_columns:
+        if is_empty(self.index_columns):
             if np.isnan(shape[0]):
                 index_value = parse_index(pd.RangeIndex(0))
             else:
@@ -238,7 +239,8 @@ def read_odps_table(
         partitions = [partitions]
 
     append_partitions = append_partitions or any(
-        pt.name in (columns or ()) for pt in (table.table_schema.partitions or ())
+        pt.name in (columns if not is_empty(columns) else ())
+        for pt in (table.table_schema.partitions or ())
     )
     op = DataFrameReadODPSTable(
         table_name=table.full_table_name,

maxframe/io/odpsio/__init__.py

@@ -19,5 +19,5 @@ from .schema import (
     odps_schema_to_pandas_dtypes,
     pandas_to_odps_schema,
 )
-from .tableio import HaloTableIO, ODPSTableIO
+from .tableio import HaloTableIO, ODPSTableIO, TunnelTableIO
 from .volumeio import ODPSVolumeReader, ODPSVolumeWriter

maxframe/io/odpsio/arrow.py

@@ -14,10 +14,12 @@
 
 from typing import Any, Tuple, Union
 
+import numpy as np
 import pandas as pd
 import pyarrow as pa
 
 from ...core import OutputType
+from ...lib.version import parse as parse_version
 from ...protocol import DataFrameTableMeta
 from ...tensor.core import TENSOR_TYPE
 from ...typing_ import ArrowTableType, PandasObjectTypes
@@ -109,7 +111,26 @@ def pandas_to_arrow(
             df = pd.DataFrame([[df]], columns=names)
     else:  # this could never happen  # pragma: no cover
         raise ValueError(f"Does not support meta type {table_meta.type!r}")
-    pa_table = pa.Table.from_pandas(df, nthreads=nthreads, preserve_index=False)
+
+    try:
+        pa_table = pa.Table.from_pandas(df, nthreads=nthreads, preserve_index=False)
+    except pa.ArrowTypeError as ex:  # pragma: no cover
+        late_np_version = parse_version(np.__version__) >= parse_version("1.20")
+        early_pa_version = parse_version(pa.__version__) <= parse_version("4.0")
+        if (
+            late_np_version
+            and early_pa_version
+            and "Did not pass numpy.dtype object" in str(ex)
+        ):
+            raise TypeError(
+                "Potential dependency conflict. Try update to pyarrow>4.0 "
+                "or downgrade to numpy<1.20. Details can be seen at "
+                "https://github.com/numpy/numpy/issues/17913. "
+                f"Raw error message: {ex!r}"
+            ).with_traceback(ex.__traceback__) from None
+        else:
+            raise
+
     if table_datetime_cols:
         col_names = pa_table.schema.names
         col_datas = []

maxframe/io/odpsio/tableio.py

@@ -15,6 +15,7 @@
 import os
 import time
 from abc import ABC, abstractmethod
+from collections import OrderedDict
 from contextlib import contextmanager
 from typing import Dict, List, Optional, Union
 
@@ -25,7 +26,7 @@ from odps.apis.storage_api import (
     TableBatchScanResponse,
     TableBatchWriteResponse,
 )
-from odps.tunnel import TableTunnel
+from odps.tunnel import TableDownloadSession, TableDownloadStatus, TableTunnel
 from odps.types import OdpsSchema, PartitionSpec, timestamp_ntz
 from odps.utils import call_with_retry
 
@@ -36,12 +37,13 @@ except ImportError:
 
 from ...config import options
 from ...env import ODPS_STORAGE_API_ENDPOINT
-from ...utils import sync_pyodps_options
+from ...utils import is_empty, sync_pyodps_options
 from .schema import odps_schema_to_arrow_schema
 
 PartitionsType = Union[List[str], str, None]
 
 _DEFAULT_ROW_BATCH_SIZE = 4096
+_DOWNLOAD_ID_CACHE_SIZE = 100
 
 
 class ODPSTableIO(ABC):
@@ -65,7 +67,11 @@ class ODPSTableIO(ABC):
     ) -> OdpsSchema:
         final_cols = []
 
-        columns = columns or [col.name for col in table_schema.simple_columns]
+        columns = (
+            columns
+            if not is_empty(columns)
+            else [col.name for col in table_schema.simple_columns]
+        )
         if partition_columns is True:
             partition_columns = [c.name for c in table_schema.partitions]
         else:
@@ -215,6 +221,46 @@ class TunnelMultiPartitionReader:
 
 
 class TunnelTableIO(ODPSTableIO):
+    _down_session_ids = OrderedDict()
+
+    @classmethod
+    def create_download_sessions(
+        cls,
+        odps_entry: ODPS,
+        full_table_name: str,
+        partitions: List[Optional[str]] = None,
+    ) -> Dict[Optional[str], TableDownloadSession]:
+        table = odps_entry.get_table(full_table_name)
+        tunnel = TableTunnel(odps_entry)
+        parts = (
+            [partitions]
+            if partitions is None or isinstance(partitions, str)
+            else partitions
+        )
+        part_to_session = dict()
+        for part in parts:
+            part_key = (full_table_name, part)
+            down_session = None
+
+            if part_key in cls._down_session_ids:
+                down_id = cls._down_session_ids[part_key]
+                down_session = tunnel.create_download_session(
+                    table, async_mode=True, partition_spec=part, download_id=down_id
+                )
+                if down_session.status != TableDownloadStatus.Normal:
+                    down_session = None
+
+            if down_session is None:
+                down_session = tunnel.create_download_session(
+                    table, async_mode=True, partition_spec=part
+                )
+
+            while len(cls._down_session_ids) >= _DOWNLOAD_ID_CACHE_SIZE:
+                cls._down_session_ids.popitem(False)
+            cls._down_session_ids[part_key] = down_session.id
+            part_to_session[part] = down_session
+        return part_to_session
+
     @contextmanager
     def open_reader(
         self,
@@ -241,21 +287,15 @@ class TunnelTableIO(ODPSTableIO):
             or (reverse_range and start is None)
         ):
             with sync_pyodps_options():
-                table = self._odps.get_table(full_table_name)
-                tunnel = TableTunnel(self._odps)
-                parts = (
-                    [partitions]
-                    if partitions is None or isinstance(partitions, str)
-                    else partitions
+                tunnel_sessions = self.create_download_sessions(
+                    self._odps, full_table_name, partitions
+                )
+                part_to_down_id = {
+                    pt: session.id for (pt, session) in tunnel_sessions.items()
+                }
+                total_records = sum(
+                    session.count for session in tunnel_sessions.values()
                 )
-                part_to_down_id = dict()
-                total_records = 0
-                for part in parts:
-                    down_session = tunnel.create_download_session(
-                        table, async_mode=True, partition_spec=part
-                    )
-                    part_to_down_id[part] = down_session.id
-                    total_records += down_session.count
 
         count = None
         if start is not None or stop is not None:

maxframe/serialization/serializables/core.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 import weakref
-from collections import defaultdict
+from collections import OrderedDict
 from typing import Any, Dict, List, Optional, Tuple, Type
 
 import msgpack
@@ -98,14 +98,18 @@ class SerializableMeta(type):
                 non_primitive_fields.append(v)
 
         # count number of fields for every base class
-        cls_to_primitive_field_count = defaultdict(lambda: 0)
-        cls_to_non_primitive_field_count = defaultdict(lambda: 0)
+        cls_to_primitive_field_count = OrderedDict()
+        cls_to_non_primitive_field_count = OrderedDict()
         for field_name in field_order:
             cls_hash = field_to_cls_hash[field_name]
             if field_name in primitive_field_names:
-                cls_to_primitive_field_count[cls_hash] += 1
+                cls_to_primitive_field_count[cls_hash] = (
+                    cls_to_primitive_field_count.get(cls_hash, 0) + 1
+                )
             else:
-                cls_to_non_primitive_field_count[cls_hash] += 1
+                cls_to_non_primitive_field_count[cls_hash] = (
+                    cls_to_non_primitive_field_count.get(cls_hash, 0) + 1
+                )
 
         slots = set(properties.pop("__slots__", set()))
         slots.update(properties_field_slot_names)
@@ -120,9 +124,11 @@ class SerializableMeta(type):
         properties["_FIELD_ORDER"] = field_order
         properties["_FIELD_TO_NAME_HASH"] = field_to_cls_hash
         properties["_PRIMITIVE_FIELDS"] = primitive_fields
-        properties["_CLS_TO_PRIMITIVE_FIELD_COUNT"] = dict(cls_to_primitive_field_count)
+        properties["_CLS_TO_PRIMITIVE_FIELD_COUNT"] = OrderedDict(
+            cls_to_primitive_field_count
+        )
         properties["_NON_PRIMITIVE_FIELDS"] = non_primitive_fields
-        properties["_CLS_TO_NON_PRIMITIVE_FIELD_COUNT"] = dict(
+        properties["_CLS_TO_NON_PRIMITIVE_FIELD_COUNT"] = OrderedDict(
             cls_to_non_primitive_field_count
         )
         properties["__slots__"] = tuple(slots)
@@ -296,21 +302,51 @@ class SerializableSerializer(Serializer):
             else:
                 field.set(obj, value)
 
+    @classmethod
+    def _prune_server_fields(
+        cls,
+        client_cls_to_field_count: Optional[Dict[int, int]],
+        server_cls_to_field_count: Dict[int, int],
+        server_fields: list,
+    ) -> list:
+        if not client_cls_to_field_count:  # pragma: no cover
+            # todo remove this branch when all versions below v0.1.0b5 is eliminated
+            return server_fields
+        if set(client_cls_to_field_count.keys()) == set(
+            server_cls_to_field_count.keys()
+        ):
+            return server_fields
+        ret_server_fields = []
+        server_pos = 0
+        for cls_hash, count in server_cls_to_field_count.items():
+            if cls_hash in client_cls_to_field_count:
+                ret_server_fields.extend(server_fields[server_pos : server_pos + count])
+            server_pos += count
+        return ret_server_fields
+
     @classmethod
     def _set_field_values(
         cls,
         obj: Serializable,
         values: List[Any],
-        client_cls_to_field_count: Optional[Dict[str, int]],
+        client_cls_to_field_count: Optional[Dict[int, int]],
         is_primitive: bool = True,
     ):
         obj_class = type(obj)
         if is_primitive:
             server_cls_to_field_count = obj_class._CLS_TO_PRIMITIVE_FIELD_COUNT
-            server_fields = obj_class._PRIMITIVE_FIELDS
+            server_fields = cls._prune_server_fields(
+                client_cls_to_field_count,
+                server_cls_to_field_count,
+                obj_class._PRIMITIVE_FIELDS,
+            )
         else:
             server_cls_to_field_count = obj_class._CLS_TO_NON_PRIMITIVE_FIELD_COUNT
-            server_fields = obj_class._NON_PRIMITIVE_FIELDS
+            server_fields = cls._prune_server_fields(
+                client_cls_to_field_count,
+                server_cls_to_field_count,
+                obj_class._NON_PRIMITIVE_FIELDS,
+            )
 
         legacy_to_new_hash = {
             c._LEGACY_NAME_HASH: c._NAME_HASH

maxframe/serialization/serializables/tests/test_serializable.py

@@ -221,7 +221,10 @@ def test_compatible_serializable(set_is_ci):
             _ref_val = ReferenceField("ref_val", "MySimpleSerializable")
             _dict_val = DictField("dict_val")
 
-        class MySubSerializable(MySimpleSerializable):
+        class MyMidSerializable(MySimpleSerializable):
+            _i_bool_val = Int64Field("i_bool_val", default=True)
+
+        class MySubSerializable(MyMidSerializable):
             _m_int_val = Int64Field("m_int_val", default=250)
             _m_str_val = StringField("m_str_val", default="SUB_STR")
 

maxframe/tensor/merge/vstack.py

@@ -48,7 +48,7 @@ def vstack(tup):
 
     Examples
     --------
-    >>> import mars.tensor as mt
+    >>> import maxframe.tensor as mt
 
     >>> a = mt.array([1, 2, 3])
     >>> b = mt.array([2, 3, 4])

maxframe/utils.py

@@ -1127,3 +1127,9 @@ def sync_pyodps_options():
 
 def str_to_bool(s: Optional[str]) -> Optional[bool]:
     return s.lower().strip() in ("true", "1") if s is not None else None
+
+
+def is_empty(val):
+    if isinstance(val, (pd.DataFrame, pd.Series, pd.Index)):
+        return val.empty
+    return not bool(val)

{maxframe-1.1.0.dist-info → maxframe-1.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maxframe
-Version: 1.1.0
+Version: 1.1.1
 Summary: MaxFrame operator-based data analyze framework
 Requires-Dist: numpy<2.0.0,>=1.19.0
 Requires-Dist: pandas>=1.0.0
@@ -11,11 +11,11 @@ Requires-Dist: msgpack>=1.0.0
 Requires-Dist: traitlets>=5.0
 Requires-Dist: cloudpickle<3.0.0,>=1.5.0
 Requires-Dist: pyyaml>=5.1
+Requires-Dist: pickle5; python_version < "3.8"
 Requires-Dist: tornado>=6.0
 Requires-Dist: defusedxml>=0.5.0
 Requires-Dist: tqdm>=4.1.0
-Requires-Dist: importlib-metadata>=1.4
-Requires-Dist: pickle5; python_version < "3.8"
+Requires-Dist: importlib_metadata>=1.4
 Provides-Extra: dev
 Requires-Dist: black>=22.3.0; extra == "dev"
 Requires-Dist: flake8>=5.0.4; extra == "dev"

{maxframe-1.1.0.dist-info → maxframe-1.1.1.dist-info}/RECORD

@@ -1,32 +1,32 @@
-maxframe-1.1.0.dist-info/RECORD,,
-maxframe-1.1.0.dist-info/WHEEL,sha256=9P88gEK1pHQvFDZpM7eNQklapMkn3JPT4pYB8yyZkL8,111
-maxframe-1.1.0.dist-info/top_level.txt,sha256=64x-fc2q59c_vXwNUkehyjF1vb8JWqFSdYmUqIFqoTM,31
-maxframe-1.1.0.dist-info/METADATA,sha256=n6x3IidqbCnXPjnT-Q0Mtso0YWC4O5ndKVdbt_n-wEg,3022
+maxframe-1.1.1.dist-info/RECORD,,
+maxframe-1.1.1.dist-info/WHEEL,sha256=dqcMvds7ym-Clu_2fY2xyUvpe9ORKPgCJKUqdwjOvr8,110
+maxframe-1.1.1.dist-info/top_level.txt,sha256=64x-fc2q59c_vXwNUkehyjF1vb8JWqFSdYmUqIFqoTM,31
+maxframe-1.1.1.dist-info/METADATA,sha256=Uy7LPFzY8VsjVyHUwHA-dqnSN6Jk2NWdWef35C-Vyik,3022
 maxframe_client/conftest.py,sha256=7cwy2sFy5snEaxvtMvxfYFUnG6WtYC_9XxVrwJxOpcU,643
 maxframe_client/__init__.py,sha256=0_6MYIqksNc-B0hORLb0yqNQUhtqdFD7TGg39bQ-_NI,689
-maxframe_client/fetcher.py,sha256=HTp7U2bv9jj-lc67jH2R0QG7MD1LygvfZi1Tg9nlefs,9203
+maxframe_client/fetcher.py,sha256=6gAKBI23GpkrgKyveHZlABYMWbzgiKwE7W4UWKhrzAI,9172
 maxframe_client/clients/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
 maxframe_client/clients/framedriver.py,sha256=-Ux7Q_bWuUSG8r14u84-1UVT9V08q_z4jGxV8kvPQaI,4557
 maxframe_client/tests/test_session.py,sha256=XdbWE3jzmzphsPmbAk5L8xYVUnAXarDDzVhVYIWwnjE,11196
 maxframe_client/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
 maxframe_client/tests/test_fetcher.py,sha256=7mVDO456wcUMajguTJ4FWkSpuonLfinFfiz31ZYnHNs,4158
-maxframe_client/session/task.py,sha256=qjCy-2jsLSDdy1_9T23RgYLD-JLEmu36eV-9FWEaPkM,11858
+maxframe_client/session/task.py,sha256=eL_bQShBAtUttz13JyBbDkMv9UIpnL6wSk9qtwmUNqk,12027
 maxframe_client/session/graph.py,sha256=rRilIWsiVfj_N160s8uv2s7mi_nhx7JxSa9BkhyLRnE,4376
 maxframe_client/session/__init__.py,sha256=KPqhSlAJiuUz8TC-z5o7mHDVXzLSqWwrZ33zNni7piY,832
 maxframe_client/session/consts.py,sha256=kQv67i4wyhV2ZQXwJf_5k4PRXhN811LmYoo2C3NB7tk,1391
-maxframe_client/session/odps.py,sha256=QbOvlQ57xog-EGZM9nXpzdZjNUUKsYOSto54Edgfv2A,24161
-maxframe_client/session/tests/test_task.py,sha256=jtUBLyZYDedpe_ZSUGWs4_WFIdWBwYvP9ywmvd3N1jw,3142
+maxframe_client/session/odps.py,sha256=8PbU-5jNhFO2psLQ0X_kiiqKcZXwgeshxZkoh6poMvA,24928
+maxframe_client/session/tests/test_task.py,sha256=L1t8IPy9p60vuLgjdQym2ADSa9LI6cQArJ0pnXxtvS0,4701
 maxframe_client/session/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
 maxframe/_utils.pyx,sha256=I4kmfhNr-xXK2ak22dr4Vwahzn-JmTaYctbL3y9_UBQ,17017
-maxframe/conftest.py,sha256=H6KND9MzHGR7WKlP35SeFE9BNafXxK7XtUUpQPN28BE,6057
-maxframe/_utils.cpython-311-darwin.so,sha256=9qAi_0O-uleuNpcsWyxec_1pW4ZNS3vkcApi5WNehqg,415312
+maxframe/conftest.py,sha256=s0GB7iP-m8UTSrYF0P1TH-kffzhZsch7EaV2KObVZ8k,6293
+maxframe/_utils.cpython-311-darwin.so,sha256=Oq4_skHnHCdPIW6TFHRfEzZ5IiFTrpyuRthEUps1_WE,415312
 maxframe/opcodes.py,sha256=Ef_CJvPvMi_JO4CYQfWRc_eZnMKtoiJWnO7eC-xNAOg,10348
 maxframe/env.py,sha256=_K499f7giN7Iu9f39iI9p_naaEDoJ0rx8dInbzqFOVI,1402
 maxframe/mixin.py,sha256=HBAeWYGb7N6ZIgkA-YpkKiSY1GetcEVNTuMb0ieznBs,3524
 maxframe/protocol.py,sha256=kP8dnBhQEI6BcVFn2uuZZTmvr4xJA-R-SZi9ZJ_iqtY,18984
 maxframe/session.py,sha256=FFciufr4jyKKWBDfbd_OwHGGT8zpRz_8rkZHrGhJMNM,36393
 maxframe/__init__.py,sha256=SqTFS_1o2HDuVY1mhS0ELlqDuM-biwM_MN0EYGkJLf0,1004
-maxframe/utils.py,sha256=f4iDtyZG7Wamm7jPU5Tse2RwguXJz5KOIiDAgycyW0o,34550
+maxframe/utils.py,sha256=eubuRneXZ_fedIwMDLChgAq0WAXMuFjjyLtc4r_7zjg,34682
 maxframe/extension.py,sha256=F5XTYzW5hNw0AIQz3d6u6Yk7adDdiV4c-HD7bF0X1FI,2659
 maxframe/errors.py,sha256=vHcpVrKRHmoZPa6IwsdDT-jOZUTlhCp8c0e8F2C-5uU,966
 maxframe/udf.py,sha256=HrZzDSNHmv63lCt4bMoPPPVV0HdzIKPL89khmj5yAAc,5157
@@ -84,10 +84,10 @@ maxframe/dataframe/datasource/from_index.py,sha256=2061zsQn-BhyHTT0X9tE0JK8vLxQU
 maxframe/dataframe/datasource/dataframe.py,sha256=LxAKF4gBIHhnJQPuaAUdIEyMAq7HTfiEeNVls5n4I4A,2023
 maxframe/dataframe/datasource/series.py,sha256=QcYiBNcR8jjH6vdO6l6H9F46KHmlBqVCTI2tv9eyZ9w,1909
 maxframe/dataframe/datasource/__init__.py,sha256=C8EKsHTJi-1jvJUKIpZtMtsK-ZID3dtxL1voXnaltTs,640
-maxframe/dataframe/datasource/read_odps_query.py,sha256=vA95FxGZHAKE7KmEespmOEF6wFP9fWeXxle2zQ77PF0,13468
+maxframe/dataframe/datasource/read_odps_query.py,sha256=eOYQdYEIdfQ9SqHkNbY_OxBdUVuH-KZNLDA7fxp8erk,13504
 maxframe/dataframe/datasource/core.py,sha256=ozFmDgw1og7nK9_jU-u3tLEq9pNbitN-8w8XWdbKkJ0,2687
 maxframe/dataframe/datasource/date_range.py,sha256=8JMr_Ife5pKCS_ca7W50Fyoc1JigOJirVzdVaPDzeFo,17227
-maxframe/dataframe/datasource/read_odps_table.py,sha256=T44rPm2-2plSCghTGX0U9mnTrQgpjczHt0DPt2H5HM8,9338
+maxframe/dataframe/datasource/read_odps_table.py,sha256=E2hv-9CxpXlxf0r_lc_G6Cs-voYQ1BunuC9A28K5Sx4,9409
 maxframe/dataframe/datasource/read_parquet.py,sha256=9auOcy8snTxCOohgXZCUXfT_O39irdkBngZH5svgx0E,14531
 maxframe/dataframe/datasource/from_tensor.py,sha256=4viuN5SLLye7Xeb8kouOpm-osoQ2yEovWTDNPQuW8gE,14727
 maxframe/dataframe/datasource/from_records.py,sha256=WBYouYyg7m_8NJdN-yUWSfJlIpm6DVP3IMfLXZFugyI,3442
@@ -306,7 +306,7 @@ maxframe/core/entity/objects.py,sha256=EnS0F2ageFTEvNfylqp21LTfTv3Q3p0X0Ur6HgWr2
 maxframe/core/entity/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
 maxframe/core/entity/tests/test_objects.py,sha256=-HS2dux6OdmwtMY6GRxwASq2gPSRY5S9YwTqeG-dGaA,1349
 maxframe/core/graph/__init__.py,sha256=tqUUWDOXp2KFjO7zv1dN_-ttE-ef09-S1GNt8EQ35Bk,765
-maxframe/core/graph/core.cpython-311-darwin.so,sha256=Lc0I9Vg7IdEOrrF7zTdu5DWsvUoJQq0aNA4hMge3KqI,321272
+maxframe/core/graph/core.cpython-311-darwin.so,sha256=v7p_bEegj250B3qhBqX4wtt7QWslK7PeOgmLrbdL-EI,321272
 maxframe/core/graph/entity.py,sha256=3ifzsEDIxzDjeM9MhlSwgz92GuaEEoCVWxEkEu2xIgE,4863
 maxframe/core/graph/core.pyx,sha256=kyqE5-X9Tc82wU4N_zsf8jNthAHWHTVRNFQWNNbzgpM,15923
 maxframe/core/graph/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
@@ -326,12 +326,12 @@ maxframe/core/operator/objects.py,sha256=lEicZmIsR2sgWsH3oAeeYMde8C0TYeu2vkAyGj8
 maxframe/core/operator/base.py,sha256=IXBJ0Nd8JAcnwN8FQ1H-QXiMRKAz2_LUZstku7Msv1E,15336
 maxframe/core/operator/tests/test_core.py,sha256=57aICnc5VLqdVK7icAORTWC81bSjBxeeVWIJcha9J_0,1691
 maxframe/core/operator/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
-maxframe/config/config.py,sha256=xFJWljdDYZpTGixRn58NWKXKzaXvm-OPF2QONYPHns0,15510
+maxframe/config/config.py,sha256=3E7VsRV-IVJ8lu0SDU4NMdizuJBVEu0uh_uUFIFOzBU,16280
 maxframe/config/validators.py,sha256=UjbxMKZcDG98-9uCQESm_V56d-VUD7kQGV0KJghVbj8,2511
-maxframe/config/__init__.py,sha256=g5lN3nP2HTAXa6ExGxU1NwU1M9ulYPmAcsV-gU7nIW8,656
+maxframe/config/__init__.py,sha256=mkW-3nDoNFlRIECv2WbZBv71of2X4KmTUpXK2zuUFjg,683
 maxframe/config/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
 maxframe/config/tests/test_validators.py,sha256=U_7yKSl0FdVdDwKU1EsnCzNWaOXi8xrIC08x7hb_O4c,1240
-maxframe/config/tests/test_config.py,sha256=Cx3buKli4F77zwJaB-vnUSlE9LUbt3ObHW38cIE2dDs,2736
+maxframe/config/tests/test_config.py,sha256=iddxMP_OfRtcgVtXi42VanLF7CkDy_IwASDVZA9PHds,3283
 maxframe/serialization/exception.py,sha256=2Ubi2ld5XpduqAln26q0T67XLj-OpdwW1Z-nyC_wtCI,2993
 maxframe/serialization/core.pxd,sha256=eBrSXiAPlX83Kbwml5IKJvqsIT03lPCeS6is9HplLiU,1501
 maxframe/serialization/pandas.py,sha256=56qngiE78mwHw7srbDE_rnN1XxgX34B7wdAhDJQRfaE,8488
@@ -341,21 +341,21 @@ maxframe/serialization/__init__.py,sha256=LrwesIKJ6MR_mhxW7qRXJXohH9waubZMR9-Yic
 maxframe/serialization/maxframe_objects.py,sha256=R9WEjbHL0Kr56OGkYDU9fcGi7gII6fGlXhi6IyihTsM,1365
 maxframe/serialization/numpy.py,sha256=8_GSo45l_eNoMn4NAGEb9NLXY_9i4tf9KK4EzG0mKpA,3213
 maxframe/serialization/scipy.py,sha256=hP0fAW0di9UgJrGtANB2S8hLDbFBtR8p5NDqAMt5rDI,2427
-maxframe/serialization/core.cpython-311-darwin.so,sha256=zcixQdl7H6lqnqhRaRWiZKJotnFQnD24A14aACQULFs,616072
+maxframe/serialization/core.cpython-311-darwin.so,sha256=qZSAT9mxY85c87o7paHXik1AbkBMqLilrE_6jKClSTw,616072
 maxframe/serialization/core.pyx,sha256=mqfb7YUd-Vop8wmGJTjziDP59YX2tr1iN6puRmFI7dg,35551
 maxframe/serialization/tests/test_serial.py,sha256=fL1ufMU7Lf1fgQ4fwJ0QrKWGQIsw_zHtAQ9zkRfFrOI,12543
 maxframe/serialization/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
 maxframe/serialization/serializables/field.py,sha256=atVgX-9rsVG1fTev7vjQArVwIEaCRjXoSEjpQ3mh6bA,16015
 maxframe/serialization/serializables/__init__.py,sha256=_wyFZF5QzSP32wSXlXHEPl98DN658I66WamP8XPJy0c,1351
-maxframe/serialization/serializables/core.py,sha256=gP83bHStfjJf9zuBVBCl40k6wAGAyQ4rLuFdUiLvbxA,16156
+maxframe/serialization/serializables/core.py,sha256=drZB6nSOi7P-W-bbZpH2PirlSJNreq44PdlKI6qSEFQ,17509
 maxframe/serialization/serializables/field_type.py,sha256=tgaLzbJ9RmzPOkL_iOfl9E8njZ5J7MtRkGLDnY0lRz8,14933
 maxframe/serialization/serializables/tests/test_field_type.py,sha256=T3ebXbUkKveC9Pq1nIl85e4eYascFeJ52d0REHbz5jo,4381
 maxframe/serialization/serializables/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
-maxframe/serialization/serializables/tests/test_serializable.py,sha256=Yu_38DYeGlm7AH8BnkJzddFtmBfTt7OV8DcFMEfFrI8,10148
+maxframe/serialization/serializables/tests/test_serializable.py,sha256=wpdqiKnMYpQm0ztbJqMF7-vFnjYe4zWiSMXJnAFGt3I,10266
 maxframe/io/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
-maxframe/io/odpsio/tableio.py,sha256=zgk7v_Dcq66u0mtBnF7J9ZoeGurEajsXaxgkRF4u9-o,21055
-maxframe/io/odpsio/arrow.py,sha256=L7b6opJok-tM2t9KQS3iQX-WRnbCJZUNnSUQkHR7KkQ,5108
-maxframe/io/odpsio/__init__.py,sha256=VD_mvxljDKDfJHYuN3OpjJWzHqFT5DKykIvECahGu0w,902
+maxframe/io/odpsio/tableio.py,sha256=Imt5_o53Lg0kqJwv0rdIkIDkhimU4BPYoyHzFKXfTS8,22452
+maxframe/io/odpsio/arrow.py,sha256=Fq5LFyiu8zprCoRs8ITWEn8pIFCZzoYZrL4IDreYURQ,5929
+maxframe/io/odpsio/__init__.py,sha256=GVR_nKbpwG7DTEp2oOoNb2lMWudIJBIAQfdQQ_2Meh4,917
 maxframe/io/odpsio/volumeio.py,sha256=y3JwUgBryoBFrkPppT7hCf6VGyJDADM0MUlBXmcia5w,2948
 maxframe/io/odpsio/schema.py,sha256=U6kHDkRnp3uluMN36ojPQ1SzfiErfdcphJvxUYejQZQ,12980
 maxframe/io/odpsio/tests/test_tableio.py,sha256=2yuQnzmgDZpDlhSrLjOcOWnnnRZrTfYcCMhvCQOoa7Q,5746
@@ -378,7 +378,7 @@ maxframe/lib/version.py,sha256=yQ6HkDOvU9X1rpI49auh-qku2g7gIiztgEH6v1urOrk,18321
 maxframe/lib/compression.py,sha256=k9DSrl_dNBsn5azLjBdL5B4WZ6eNvmCrdMbcF1G7JSc,1442
 maxframe/lib/__init__.py,sha256=CzfbLNqqm1yR1i6fDwCd4h1ptuKVDbURFVCb0ra7QNc,642
 maxframe/lib/mmh3.pyi,sha256=AOp_XqbA5-NwepeeBeG0OFJj5tjEAFLzcViyRNZ0eVI,1494
-maxframe/lib/mmh3.cpython-311-darwin.so,sha256=4hWIwNr6dYjCkYCURWSonDNxC0wCqu_1FqtZaFWCgFY,37256
+maxframe/lib/mmh3.cpython-311-darwin.so,sha256=yiSubwaj3nCtmUZcr7pOl3CkP0e1wrlTqNVMCxuXMiA,37256
 maxframe/lib/functools_compat.py,sha256=PMSkct9GIbzq-aBwTnggrOLNfLh4xQnYTIFMPblzCUA,2616
 maxframe/lib/mmh3_src/mmh3module.cpp,sha256=9J9eA42eKWTl546fvfQPNuIM3B2jpWSADpgIw3tr2jg,11604
 maxframe/lib/mmh3_src/MurmurHash3.h,sha256=lg5uXUFyMBge2BWRn0FgrqaCFCMfDWoTXD4PQtjHrMA,1263
@@ -466,7 +466,7 @@ maxframe/tensor/datasource/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQB
 maxframe/tensor/rechunk/__init__.py,sha256=pQ4vh6rNzxjIkxrWTxXjAcTDmUw3961-H1f__-lJl6A,797
 maxframe/tensor/rechunk/rechunk.py,sha256=ltvGlUQxzoHSE7bC6J6uQ8sO-YhuFpxmj8-ArJJXIoY,1392
 maxframe/tensor/merge/concatenate.py,sha256=q0qVpizcU7E6op6D54bF4bl2eMLIJOOwb8AkG1jrDCE,3213
-maxframe/tensor/merge/vstack.py,sha256=FzuynVGKkNiOXbzp4Jkde0VMxzs_FfxkxO7J0UU9nhs,2264
+maxframe/tensor/merge/vstack.py,sha256=XGyubfPwkauUQ2zw4D9qd5uLLSRw5LgadHBpEQH71V8,2268
 maxframe/tensor/merge/__init__.py,sha256=NCuoHVwBtVUQnl8-0ph9cT5ARRn3pTdqZt76-XSvpvs,686
 maxframe/tensor/merge/stack.py,sha256=4ZMVqtJQ0nQtbtnJKTX0Z_fSUn8slLLSO56mFT-b0gE,4145
 maxframe/tensor/merge/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596

{maxframe-1.1.0.dist-info → maxframe-1.1.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.45.0)
+Generator: setuptools (75.6.0)
 Root-Is-Purelib: false
 Tag: cp311-cp311-macosx_10_9_x86_64
 

maxframe_client/fetcher.py

@@ -20,7 +20,6 @@ import pandas as pd
 import pyarrow as pa
 from odps import ODPS
 from odps.models import ExternalVolume
-from odps.tunnel import TableTunnel
 
 from maxframe.core import OBJECT_TYPE
 from maxframe.dataframe.core import DATAFRAME_TYPE
@@ -28,6 +27,7 @@ from maxframe.io.objects import get_object_io_handler
 from maxframe.io.odpsio import (
     ODPSTableIO,
     ODPSVolumeReader,
+    TunnelTableIO,
     arrow_to_pandas,
     build_dataframe_table_meta,
     odps_schema_to_pandas_dtypes,
@@ -136,11 +136,10 @@ class ODPSTableFetcher(ToThreadMixin, ResultFetcher):
                     dtypes = odps_schema_to_pandas_dtypes(table.table_schema)
                     tileable.refresh_from_dtypes(dtypes)
 
-                tunnel = TableTunnel(self._odps_entry)
-                total_records = 0
-                for part_spec in part_specs:
-                    session = tunnel.create_download_session(table, part_spec)
-                    total_records += session.count
+                part_sessions = TunnelTableIO.create_download_sessions(
+                    self._odps_entry, info.full_table_name, part_specs
+                )
+                total_records = sum(session.count for session in part_sessions.values())
 
             new_shape_list = list(tileable.shape)
             new_shape_list[0] = total_records

maxframe_client/session/odps.py

@@ -84,10 +84,21 @@ class MaxFrameServiceCaller(metaclass=abc.ABCMeta):
     def get_settings_to_upload(self) -> Dict[str, Any]:
         sql_settings = (odps_options.sql.settings or {}).copy()
         sql_settings.update(options.sql.settings or {})
-
         quota_name = options.session.quota_name or getattr(
             odps_options, "quota_name", None
         )
+        quota_settings = {
+            sql_settings.get("odps.task.wlm.quota", None),
+            options.spe.task.settings.get("odps.task.wlm.quota", None),
+            options.pythonpack.task.settings.get("odps.task.wlm.quota", None),
+            quota_name,
+        }.difference([None])
+        if len(quota_settings) >= 2:
+            raise ValueError(
+                "Quota settings are conflicting: %s" % ", ".join(sorted(quota_settings))
+            )
+        elif len(quota_settings) == 1:
+            quota_name = quota_settings.pop()
         lifecycle = options.session.table_lifecycle or odps_options.lifecycle
         temp_lifecycle = (
             options.session.temp_table_lifecycle or odps_options.temp_lifecycle
@@ -332,6 +343,11 @@ class MaxFrameSession(ToThreadMixin, IsolatedAsyncSession):
             self._last_settings = copy.deepcopy(new_settings)
             return new_settings
 
+        if self._last_settings.get("session.quota_name", None) != new_settings.get(
+            "session.quota_name", None
+        ):
+            raise ValueError("Quota name cannot be changed after sessions are created")
+
         update = dict()
         for k in new_settings.keys():
             old_item = self._last_settings.get(k)

maxframe_client/session/task.py

@@ -126,10 +126,13 @@ class MaxFrameInstanceCaller(MaxFrameServiceCaller):
 
     def _create_maxframe_task(self) -> MaxFrameTask:
         task = MaxFrameTask(name=self._task_name, major_version=self._major_version)
+        mf_settings = self.get_settings_to_upload()
         mf_opts = {
-            "odps.maxframe.settings": json.dumps(self.get_settings_to_upload()),
+            "odps.maxframe.settings": json.dumps(mf_settings),
             "odps.maxframe.output_format": self._output_format,
         }
+        if mf_settings.get("session.quota_name", None):
+            mf_opts["odps.task.wlm.quota"] = mf_settings["session.quota_name"]
         if mf_version:
             mf_opts["odps.maxframe.client_version"] = mf_version
         task.update_settings(mf_opts)

maxframe_client/session/tests/test_task.py

@@ -11,17 +11,20 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 import json
 import os
 
 import mock
+import pytest
 from defusedxml import ElementTree
 from odps import ODPS
 from odps import options as odps_options
 
+from maxframe import options
+from maxframe.config import option_context
+
 from ...session.consts import MAXFRAME_OUTPUT_JSON_FORMAT
-from ...session.task import MaxFrameInstanceCaller, MaxFrameTask
+from ...session.task import MaxFrameInstanceCaller, MaxFrameTask, MaxFrameTaskSession
 
 expected_file_dir = os.path.join(os.path.dirname(__file__), "expected-data")
 
@@ -79,3 +82,33 @@ def test_maxframe_instance_caller_creating_session():
         finally:
             odps_options.priority = old_priority
             odps_options.get_priority = old_get_priority
+
+
+@pytest.mark.asyncio
+async def test_session_quota_flag_valid():
+    def mock_create(self, task: MaxFrameTask, **kwargs):
+        assert task.properties["settings"]
+        task_settings = json.loads(task.properties["settings"])
+        assert task_settings["odps.task.wlm.quota"] == "session_quota"
+
+    with mock.patch.multiple(
+        target="maxframe_client.session.task.MaxFrameInstanceCaller",
+        _wait_instance_task_ready=mock.DEFAULT,
+        get_session=mock.DEFAULT,
+        get_logview_address=mock.DEFAULT,
+    ), mock.patch("odps.models.instances.BaseInstances.create", mock_create):
+        with option_context({"session.quota_name": "session_quota"}):
+            with pytest.raises(ValueError):
+                options.sql.settings["odps.task.wlm.quota"] = "session_quota2"
+                await MaxFrameTaskSession.init(
+                    address="test", odps_entry=ODPS.from_environments()
+                )
+            options.sql.settings["odps.task.wlm.quota"] = "session_quota"
+            mf_task_session = await MaxFrameTaskSession.init(
+                address="test", odps_entry=ODPS.from_environments()
+            )
+            with pytest.raises(ValueError):
+                options.sql.settings["odps.task.wlm.quota"] = "session_quota2"
+                mf_task_session._get_diff_settings()
+            options.sql.settings["odps.task.wlm.quota"] = "session_quota"
+            mf_task_session._get_diff_settings()