maxframe 1.1.0__tar.gz → 1.1.1__tar.gz

{maxframe-1.1.0 → maxframe-1.1.1}/PKG-INFO

@@ -1,9 +1,33 @@
 Metadata-Version: 2.1
 Name: maxframe
-Version: 1.1.0
+Version: 1.1.1
 Summary: MaxFrame operator-based data analyze framework
+Requires-Dist: numpy<2.0.0,>=1.19.0
+Requires-Dist: pandas>=1.0.0
+Requires-Dist: pyodps>=0.12.0
+Requires-Dist: scipy>=1.0
+Requires-Dist: pyarrow>=1.0.0
+Requires-Dist: msgpack>=1.0.0
+Requires-Dist: traitlets>=5.0
+Requires-Dist: cloudpickle<3.0.0,>=1.5.0
+Requires-Dist: pyyaml>=5.1
+Requires-Dist: pickle5; python_version < "3.8"
+Requires-Dist: tornado>=6.0
+Requires-Dist: defusedxml>=0.5.0
+Requires-Dist: tqdm>=4.1.0
+Requires-Dist: importlib_metadata>=1.4
 Provides-Extra: dev
+Requires-Dist: black>=22.3.0; extra == "dev"
+Requires-Dist: flake8>=5.0.4; extra == "dev"
+Requires-Dist: pre-commit>=2.15.0; extra == "dev"
+Requires-Dist: graphviz>=0.20.1; extra == "dev"
 Provides-Extra: test
+Requires-Dist: mock; extra == "test"
+Requires-Dist: pytest>=7.3.1; extra == "test"
+Requires-Dist: pytest-cov>=4.1.0; extra == "test"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "test"
+Requires-Dist: pytest-timeout>=2.1.0; extra == "test"
+Requires-Dist: matplotlib>=2.0.0; extra == "test"
 
 MaxCompute MaxFrame Client
 ==========================

{maxframe-1.1.0 → maxframe-1.1.1}/maxframe/config/__init__.py

@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .config import AttributeDict, option_context, options
+from .config import AttributeDict, option_context, options, update_wlm_quota_settings

{maxframe-1.1.0 → maxframe-1.1.1}/maxframe/config/config.py

@@ -28,6 +28,8 @@ except ImportError:
 
     available_timezones = lambda: all_timezones
 
+import logging
+
 from ..utils import get_python_tag
 from .validators import (
     ValidatorType,
@@ -43,6 +45,8 @@ from .validators import (
     is_valid_cache_path,
 )
 
+logger = logging.getLogger(__name__)
+
 _DEFAULT_REDIRECT_WARN = "Option {source} has been replaced by {target} and might be removed in a future release."
 _DEFAULT_MAX_ALIVE_SECONDS = 3 * 24 * 3600
 _DEFAULT_MAX_IDLE_SECONDS = 3600
@@ -499,3 +503,22 @@ class OptionsProxy:
 
 
 options = OptionsProxy()
+
+
+def update_wlm_quota_settings(session_id: str, engine_settings: Dict[str, Any]):
+    engine_quota = engine_settings.get("odps.task.wlm.quota", None)
+    session_quota = options.session.quota_name or None
+    if engine_quota != session_quota and engine_quota:
+        logger.warning(
+            "[Session=%s] Session quota (%s) is different to SubDag engine quota (%s)",
+            session_id,
+            session_quota,
+            engine_quota,
+        )
+        # TODO(renxiang): overwrite or not overwrite
+        return
+
+    if session_quota:
+        engine_settings["odps.task.wlm.quota"] = session_quota
+    elif "odps.task.wlm.quota" in engine_settings:
+        engine_settings.pop("odps.task.wlm.quota")

{maxframe-1.1.0 → maxframe-1.1.1}/maxframe/dataframe/datasource/read_odps_query.py

@@ -37,6 +37,7 @@ from ...serialization.serializables import (
     SeriesField,
     StringField,
 )
+from ...utils import is_empty
 from ..utils import parse_index
 from .core import ColumnPruneSupportedDataSourceMixin, IncrementalIndexDatasource
 
@@ -250,7 +251,7 @@ class DataFrameReadODPSQuery(
         self.columns = columns
 
     def __call__(self, chunk_bytes=None, chunk_size=None):
-        if not self.index_columns:
+        if is_empty(self.index_columns):
             index_value = parse_index(pd.RangeIndex(0))
         elif len(self.index_columns) == 1:
             index_value = parse_index(

{maxframe-1.1.0 → maxframe-1.1.1}/maxframe/dataframe/datasource/read_odps_table.py

@@ -34,6 +34,7 @@ from ...serialization.serializables import (
     SeriesField,
     StringField,
 )
+from ...utils import is_empty
 from ..core import DataFrame  # noqa: F401
 from ..utils import parse_index
 from .core import ColumnPruneSupportedDataSourceMixin, IncrementalIndexDatasource
@@ -76,7 +77,7 @@ class DataFrameReadODPSTable(
         self.columns = columns
 
     def __call__(self, shape, chunk_bytes=None, chunk_size=None):
-        if not self.index_columns:
+        if is_empty(self.index_columns):
             if np.isnan(shape[0]):
                 index_value = parse_index(pd.RangeIndex(0))
             else:
@@ -238,7 +239,8 @@ def read_odps_table(
         partitions = [partitions]
 
     append_partitions = append_partitions or any(
-        pt.name in (columns or ()) for pt in (table.table_schema.partitions or ())
+        pt.name in (columns if not is_empty(columns) else ())
+        for pt in (table.table_schema.partitions or ())
     )
     op = DataFrameReadODPSTable(
         table_name=table.full_table_name,

{maxframe-1.1.0 → maxframe-1.1.1}/maxframe/io/odpsio/__init__.py

@@ -19,5 +19,5 @@ from .schema import (
     odps_schema_to_pandas_dtypes,
     pandas_to_odps_schema,
 )
-from .tableio import HaloTableIO, ODPSTableIO
+from .tableio import HaloTableIO, ODPSTableIO, TunnelTableIO
 from .volumeio import ODPSVolumeReader, ODPSVolumeWriter

{maxframe-1.1.0 → maxframe-1.1.1}/maxframe/io/odpsio/arrow.py

@@ -14,10 +14,12 @@
 
 from typing import Any, Tuple, Union
 
+import numpy as np
 import pandas as pd
 import pyarrow as pa
 
 from ...core import OutputType
+from ...lib.version import parse as parse_version
 from ...protocol import DataFrameTableMeta
 from ...tensor.core import TENSOR_TYPE
 from ...typing_ import ArrowTableType, PandasObjectTypes
@@ -109,7 +111,26 @@ def pandas_to_arrow(
             df = pd.DataFrame([[df]], columns=names)
     else:  # this could never happen  # pragma: no cover
         raise ValueError(f"Does not support meta type {table_meta.type!r}")
-    pa_table = pa.Table.from_pandas(df, nthreads=nthreads, preserve_index=False)
+
+    try:
+        pa_table = pa.Table.from_pandas(df, nthreads=nthreads, preserve_index=False)
+    except pa.ArrowTypeError as ex:  # pragma: no cover
+        late_np_version = parse_version(np.__version__) >= parse_version("1.20")
+        early_pa_version = parse_version(pa.__version__) <= parse_version("4.0")
+        if (
+            late_np_version
+            and early_pa_version
+            and "Did not pass numpy.dtype object" in str(ex)
+        ):
+            raise TypeError(
+                "Potential dependency conflict. Try update to pyarrow>4.0 "
+                "or downgrade to numpy<1.20. Details can be seen at "
+                "https://github.com/numpy/numpy/issues/17913. "
+                f"Raw error message: {ex!r}"
+            ).with_traceback(ex.__traceback__) from None
+        else:
+            raise
+
     if table_datetime_cols:
         col_names = pa_table.schema.names
         col_datas = []

{maxframe-1.1.0 → maxframe-1.1.1}/maxframe/io/odpsio/tableio.py

@@ -15,6 +15,7 @@
 import os
 import time
 from abc import ABC, abstractmethod
+from collections import OrderedDict
 from contextlib import contextmanager
 from typing import Dict, List, Optional, Union
 
@@ -25,7 +26,7 @@ from odps.apis.storage_api import (
     TableBatchScanResponse,
     TableBatchWriteResponse,
 )
-from odps.tunnel import TableTunnel
+from odps.tunnel import TableDownloadSession, TableDownloadStatus, TableTunnel
 from odps.types import OdpsSchema, PartitionSpec, timestamp_ntz
 from odps.utils import call_with_retry
 
@@ -36,12 +37,13 @@ except ImportError:
 
 from ...config import options
 from ...env import ODPS_STORAGE_API_ENDPOINT
-from ...utils import sync_pyodps_options
+from ...utils import is_empty, sync_pyodps_options
 from .schema import odps_schema_to_arrow_schema
 
 PartitionsType = Union[List[str], str, None]
 
 _DEFAULT_ROW_BATCH_SIZE = 4096
+_DOWNLOAD_ID_CACHE_SIZE = 100
 
 
 class ODPSTableIO(ABC):
@@ -65,7 +67,11 @@ class ODPSTableIO(ABC):
     ) -> OdpsSchema:
         final_cols = []
 
-        columns = columns or [col.name for col in table_schema.simple_columns]
+        columns = (
+            columns
+            if not is_empty(columns)
+            else [col.name for col in table_schema.simple_columns]
+        )
         if partition_columns is True:
             partition_columns = [c.name for c in table_schema.partitions]
         else:
@@ -215,6 +221,46 @@ class TunnelMultiPartitionReader:
 
 
 class TunnelTableIO(ODPSTableIO):
+    _down_session_ids = OrderedDict()
+
+    @classmethod
+    def create_download_sessions(
+        cls,
+        odps_entry: ODPS,
+        full_table_name: str,
+        partitions: List[Optional[str]] = None,
+    ) -> Dict[Optional[str], TableDownloadSession]:
+        table = odps_entry.get_table(full_table_name)
+        tunnel = TableTunnel(odps_entry)
+        parts = (
+            [partitions]
+            if partitions is None or isinstance(partitions, str)
+            else partitions
+        )
+        part_to_session = dict()
+        for part in parts:
+            part_key = (full_table_name, part)
+            down_session = None
+
+            if part_key in cls._down_session_ids:
+                down_id = cls._down_session_ids[part_key]
+                down_session = tunnel.create_download_session(
+                    table, async_mode=True, partition_spec=part, download_id=down_id
+                )
+                if down_session.status != TableDownloadStatus.Normal:
+                    down_session = None
+
+            if down_session is None:
+                down_session = tunnel.create_download_session(
+                    table, async_mode=True, partition_spec=part
+                )
+
+            while len(cls._down_session_ids) >= _DOWNLOAD_ID_CACHE_SIZE:
+                cls._down_session_ids.popitem(False)
+            cls._down_session_ids[part_key] = down_session.id
+            part_to_session[part] = down_session
+        return part_to_session
+
     @contextmanager
     def open_reader(
         self,
@@ -241,21 +287,15 @@ class TunnelTableIO(ODPSTableIO):
             or (reverse_range and start is None)
         ):
             with sync_pyodps_options():
-                table = self._odps.get_table(full_table_name)
-                tunnel = TableTunnel(self._odps)
-                parts = (
-                    [partitions]
-                    if partitions is None or isinstance(partitions, str)
-                    else partitions
+                tunnel_sessions = self.create_download_sessions(
+                    self._odps, full_table_name, partitions
+                )
+                part_to_down_id = {
+                    pt: session.id for (pt, session) in tunnel_sessions.items()
+                }
+                total_records = sum(
+                    session.count for session in tunnel_sessions.values()
                 )
-                part_to_down_id = dict()
-                total_records = 0
-                for part in parts:
-                    down_session = tunnel.create_download_session(
-                        table, async_mode=True, partition_spec=part
-                    )
-                    part_to_down_id[part] = down_session.id
-                    total_records += down_session.count
 
         count = None
         if start is not None or stop is not None:

{maxframe-1.1.0 → maxframe-1.1.1}/maxframe/serialization/serializables/core.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 import weakref
-from collections import defaultdict
+from collections import OrderedDict
 from typing import Any, Dict, List, Optional, Tuple, Type
 
 import msgpack
@@ -98,14 +98,18 @@ class SerializableMeta(type):
                 non_primitive_fields.append(v)
 
         # count number of fields for every base class
-        cls_to_primitive_field_count = defaultdict(lambda: 0)
-        cls_to_non_primitive_field_count = defaultdict(lambda: 0)
+        cls_to_primitive_field_count = OrderedDict()
+        cls_to_non_primitive_field_count = OrderedDict()
         for field_name in field_order:
             cls_hash = field_to_cls_hash[field_name]
             if field_name in primitive_field_names:
-                cls_to_primitive_field_count[cls_hash] += 1
+                cls_to_primitive_field_count[cls_hash] = (
+                    cls_to_primitive_field_count.get(cls_hash, 0) + 1
+                )
             else:
-                cls_to_non_primitive_field_count[cls_hash] += 1
+                cls_to_non_primitive_field_count[cls_hash] = (
+                    cls_to_non_primitive_field_count.get(cls_hash, 0) + 1
+                )
 
         slots = set(properties.pop("__slots__", set()))
         slots.update(properties_field_slot_names)
@@ -120,9 +124,11 @@ class SerializableMeta(type):
         properties["_FIELD_ORDER"] = field_order
         properties["_FIELD_TO_NAME_HASH"] = field_to_cls_hash
         properties["_PRIMITIVE_FIELDS"] = primitive_fields
-        properties["_CLS_TO_PRIMITIVE_FIELD_COUNT"] = dict(cls_to_primitive_field_count)
+        properties["_CLS_TO_PRIMITIVE_FIELD_COUNT"] = OrderedDict(
+            cls_to_primitive_field_count
+        )
         properties["_NON_PRIMITIVE_FIELDS"] = non_primitive_fields
-        properties["_CLS_TO_NON_PRIMITIVE_FIELD_COUNT"] = dict(
+        properties["_CLS_TO_NON_PRIMITIVE_FIELD_COUNT"] = OrderedDict(
             cls_to_non_primitive_field_count
         )
         properties["__slots__"] = tuple(slots)
@@ -296,21 +302,51 @@ class SerializableSerializer(Serializer):
             else:
                 field.set(obj, value)
 
+    @classmethod
+    def _prune_server_fields(
+        cls,
+        client_cls_to_field_count: Optional[Dict[int, int]],
+        server_cls_to_field_count: Dict[int, int],
+        server_fields: list,
+    ) -> list:
+        if not client_cls_to_field_count:  # pragma: no cover
+            # todo remove this branch when all versions below v0.1.0b5 is eliminated
+            return server_fields
+        if set(client_cls_to_field_count.keys()) == set(
+            server_cls_to_field_count.keys()
+        ):
+            return server_fields
+        ret_server_fields = []
+        server_pos = 0
+        for cls_hash, count in server_cls_to_field_count.items():
+            if cls_hash in client_cls_to_field_count:
+                ret_server_fields.extend(server_fields[server_pos : server_pos + count])
+            server_pos += count
+        return ret_server_fields
+
     @classmethod
     def _set_field_values(
         cls,
         obj: Serializable,
         values: List[Any],
-        client_cls_to_field_count: Optional[Dict[str, int]],
+        client_cls_to_field_count: Optional[Dict[int, int]],
         is_primitive: bool = True,
     ):
         obj_class = type(obj)
         if is_primitive:
             server_cls_to_field_count = obj_class._CLS_TO_PRIMITIVE_FIELD_COUNT
-            server_fields = obj_class._PRIMITIVE_FIELDS
+            server_fields = cls._prune_server_fields(
+                client_cls_to_field_count,
+                server_cls_to_field_count,
+                obj_class._PRIMITIVE_FIELDS,
+            )
         else:
             server_cls_to_field_count = obj_class._CLS_TO_NON_PRIMITIVE_FIELD_COUNT
-            server_fields = obj_class._NON_PRIMITIVE_FIELDS
+            server_fields = cls._prune_server_fields(
+                client_cls_to_field_count,
+                server_cls_to_field_count,
+                obj_class._NON_PRIMITIVE_FIELDS,
+            )
 
         legacy_to_new_hash = {
             c._LEGACY_NAME_HASH: c._NAME_HASH

{maxframe-1.1.0 → maxframe-1.1.1}/maxframe/tensor/merge/vstack.py

@@ -48,7 +48,7 @@ def vstack(tup):
 
     Examples
     --------
-    >>> import mars.tensor as mt
+    >>> import maxframe.tensor as mt
 
     >>> a = mt.array([1, 2, 3])
     >>> b = mt.array([2, 3, 4])

{maxframe-1.1.0 → maxframe-1.1.1}/maxframe/utils.py

@@ -1127,3 +1127,9 @@ def sync_pyodps_options():
 
 def str_to_bool(s: Optional[str]) -> Optional[bool]:
     return s.lower().strip() in ("true", "1") if s is not None else None
+
+
+def is_empty(val):
+    if isinstance(val, (pd.DataFrame, pd.Series, pd.Index)):
+        return val.empty
+    return not bool(val)

{maxframe-1.1.0 → maxframe-1.1.1}/maxframe.egg-info/PKG-INFO

@@ -1,9 +1,33 @@
 Metadata-Version: 2.1
 Name: maxframe
-Version: 1.1.0
+Version: 1.1.1
 Summary: MaxFrame operator-based data analyze framework
+Requires-Dist: numpy<2.0.0,>=1.19.0
+Requires-Dist: pandas>=1.0.0
+Requires-Dist: pyodps>=0.12.0
+Requires-Dist: scipy>=1.0
+Requires-Dist: pyarrow>=1.0.0
+Requires-Dist: msgpack>=1.0.0
+Requires-Dist: traitlets>=5.0
+Requires-Dist: cloudpickle<3.0.0,>=1.5.0
+Requires-Dist: pyyaml>=5.1
+Requires-Dist: pickle5; python_version < "3.8"
+Requires-Dist: tornado>=6.0
+Requires-Dist: defusedxml>=0.5.0
+Requires-Dist: tqdm>=4.1.0
+Requires-Dist: importlib_metadata>=1.4
 Provides-Extra: dev
+Requires-Dist: black>=22.3.0; extra == "dev"
+Requires-Dist: flake8>=5.0.4; extra == "dev"
+Requires-Dist: pre-commit>=2.15.0; extra == "dev"
+Requires-Dist: graphviz>=0.20.1; extra == "dev"
 Provides-Extra: test
+Requires-Dist: mock; extra == "test"
+Requires-Dist: pytest>=7.3.1; extra == "test"
+Requires-Dist: pytest-cov>=4.1.0; extra == "test"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "test"
+Requires-Dist: pytest-timeout>=2.1.0; extra == "test"
+Requires-Dist: matplotlib>=2.0.0; extra == "test"
 
 MaxCompute MaxFrame Client
 ==========================

{maxframe-1.1.0 → maxframe-1.1.1}/maxframe_client/fetcher.py

@@ -20,7 +20,6 @@ import pandas as pd
 import pyarrow as pa
 from odps import ODPS
 from odps.models import ExternalVolume
-from odps.tunnel import TableTunnel
 
 from maxframe.core import OBJECT_TYPE
 from maxframe.dataframe.core import DATAFRAME_TYPE
@@ -28,6 +27,7 @@ from maxframe.io.objects import get_object_io_handler
 from maxframe.io.odpsio import (
     ODPSTableIO,
     ODPSVolumeReader,
+    TunnelTableIO,
     arrow_to_pandas,
     build_dataframe_table_meta,
     odps_schema_to_pandas_dtypes,
@@ -136,11 +136,10 @@ class ODPSTableFetcher(ToThreadMixin, ResultFetcher):
                     dtypes = odps_schema_to_pandas_dtypes(table.table_schema)
                     tileable.refresh_from_dtypes(dtypes)
 
-                tunnel = TableTunnel(self._odps_entry)
-                total_records = 0
-                for part_spec in part_specs:
-                    session = tunnel.create_download_session(table, part_spec)
-                    total_records += session.count
+                part_sessions = TunnelTableIO.create_download_sessions(
+                    self._odps_entry, info.full_table_name, part_specs
+                )
+                total_records = sum(session.count for session in part_sessions.values())
 
             new_shape_list = list(tileable.shape)
             new_shape_list[0] = total_records

{maxframe-1.1.0 → maxframe-1.1.1}/maxframe_client/session/odps.py

@@ -84,10 +84,21 @@ class MaxFrameServiceCaller(metaclass=abc.ABCMeta):
     def get_settings_to_upload(self) -> Dict[str, Any]:
         sql_settings = (odps_options.sql.settings or {}).copy()
         sql_settings.update(options.sql.settings or {})
-
         quota_name = options.session.quota_name or getattr(
             odps_options, "quota_name", None
         )
+        quota_settings = {
+            sql_settings.get("odps.task.wlm.quota", None),
+            options.spe.task.settings.get("odps.task.wlm.quota", None),
+            options.pythonpack.task.settings.get("odps.task.wlm.quota", None),
+            quota_name,
+        }.difference([None])
+        if len(quota_settings) >= 2:
+            raise ValueError(
+                "Quota settings are conflicting: %s" % ", ".join(sorted(quota_settings))
+            )
+        elif len(quota_settings) == 1:
+            quota_name = quota_settings.pop()
         lifecycle = options.session.table_lifecycle or odps_options.lifecycle
         temp_lifecycle = (
             options.session.temp_table_lifecycle or odps_options.temp_lifecycle
@@ -332,6 +343,11 @@ class MaxFrameSession(ToThreadMixin, IsolatedAsyncSession):
             self._last_settings = copy.deepcopy(new_settings)
             return new_settings
 
+        if self._last_settings.get("session.quota_name", None) != new_settings.get(
+            "session.quota_name", None
+        ):
+            raise ValueError("Quota name cannot be changed after sessions are created")
+
         update = dict()
         for k in new_settings.keys():
             old_item = self._last_settings.get(k)

{maxframe-1.1.0 → maxframe-1.1.1}/maxframe_client/session/task.py

@@ -126,10 +126,13 @@ class MaxFrameInstanceCaller(MaxFrameServiceCaller):
 
     def _create_maxframe_task(self) -> MaxFrameTask:
         task = MaxFrameTask(name=self._task_name, major_version=self._major_version)
+        mf_settings = self.get_settings_to_upload()
         mf_opts = {
-            "odps.maxframe.settings": json.dumps(self.get_settings_to_upload()),
+            "odps.maxframe.settings": json.dumps(mf_settings),
             "odps.maxframe.output_format": self._output_format,
         }
+        if mf_settings.get("session.quota_name", None):
+            mf_opts["odps.task.wlm.quota"] = mf_settings["session.quota_name"]
         if mf_version:
             mf_opts["odps.maxframe.client_version"] = mf_version
         task.update_settings(mf_opts)

{maxframe-1.1.0 → maxframe-1.1.1}/pyproject.toml

@@ -21,7 +21,7 @@ dependencies = [
 
 [build-system]
 requires = [
-    "setuptools<65",
+    "setuptools>=60",
     "setuptools_scm>=7.0",
     "wheel",
     "cython>=0.29",