maxframe 2.0.0b1__cp37-cp37m-win_amd64.whl → 2.2.0__cp37-cp37m-win_amd64.whl

maxframe/tensor/utils.py

@@ -167,7 +167,7 @@ def normalize_axis_tuple(axis, ndim, argname=None, allow_duplicate=False):
         except TypeError:
             pass
     # Going via an iterator directly is slower than via list comprehension.
-    axis = tuple([validate_axis(ndim, ax, argname) for ax in axis])
+    axis = tuple(validate_axis(ndim, ax, argname) for ax in axis)
     if not allow_duplicate and len(set(axis)) != len(axis):
         if argname:
             raise ValueError(f"repeated axis in `{argname}` argument")
@@ -709,8 +709,8 @@ def implement_scipy(scipy_fun_name):
     return wrapper
 
 
-def infer_scipy_dtype(scipy_fun_name):
+def infer_scipy_dtype(scipy_fun_name, **kw):
     scipy_fun = _load_scipy_func(scipy_fun_name)
     if scipy_fun is None:
         return lambda x: x
-    return infer_dtype(scipy_fun)
+    return infer_dtype(scipy_fun, **kw)

maxframe/tests/test_utils.py

@@ -32,7 +32,7 @@ import pytest
 
 from .. import utils
 from ..serialization import PickleContainer
-from ..utils import parse_size_to_megabytes
+from ..utils import parse_size_to_megabytes, validate_and_adjust_resource_ratio
 
 
 def test_string_conversion():
@@ -369,6 +369,12 @@ def test_arrow_type_from_string():
     _assert_arrow_type_convert(
         pa.struct([("key", pa.string()), ("value", pa.list_(pa.int64()))])
     )
+    _assert_arrow_type_convert(
+        pa.struct([("key", pa.string(), False), ("value", pa.list_(pa.int64()))])
+    )
+    _assert_arrow_type_convert(
+        pa.struct([("key", pa.string()), ("value", pa.list_(pa.int64()), False)])
+    )
 
 
 @pytest.mark.parametrize("use_async", [False, True])
@@ -527,6 +533,7 @@ def test_numeric_inputs_with_default_units(value, default_unit):
     "input_string, expected",
     [
         # Basic binary units
+        ("1B", 1 / BYTES_PER_MIB),
         ("1KiB", BYTES_PER_KIB / BYTES_PER_MIB),
         ("5miB", 5),
         ("2giB", 2 * BYTES_PER_GIB / BYTES_PER_MIB),
@@ -571,3 +578,38 @@ def test_parse_size_to_mega_bytes_invalid_inputs(invalid_input, default_unit):
     """Test invalid inputs that should raise ValueError"""
     with pytest.raises(ValueError):  # Catch ValueError
         parse_size_to_megabytes(invalid_input, default_number_unit=default_unit)
+
+
+@pytest.mark.parametrize(
+    "udf_resources, max_memory_cpu_ratio, adjust, expected_resources, expected_adjusted, should_warn",
+    [
+        ({"other": "value"}, 4, False, {"other": "value"}, False, False),
+        ({"memory": 8}, 4, False, {"memory": 8}, False, False),
+        ({"cpu": 2}, 4, False, {"cpu": 2}, False, False),
+        ({"cpu": 2, "memory": 2}, 4, True, {"cpu": 2, "memory": 2}, False, False),
+        ({"cpu": 2, "memory": 8}, 4, False, {"cpu": 2, "memory": 8}, False, False),
+        ({"cpu": 1, "memory": 8}, 4, False, {"cpu": 1, "memory": 8}, False, False),
+        ({"cpu": 2, "memory": 8}, 4, False, {"cpu": 2, "memory": 8}, False, False),
+        ({"cpu": 2, "memory": 8}, 4, False, {"cpu": 2, "memory": 8}, False, False),
+        ({"cpu": 1, "memory": 8}, 4, True, {"cpu": 2, "memory": 8}, True, True),
+        ({"cpu": 1, "memory": 18}, 7, True, {"cpu": 3, "memory": 18}, True, True),
+        ({"cpu": 1, "memory": 7.5}, 4, True, {"cpu": 2, "memory": 7.5}, True, True),
+    ],
+)
+def test_validate_and_adjust_resource_ratio(
+    udf_resources,
+    max_memory_cpu_ratio,
+    adjust,
+    expected_resources,
+    expected_adjusted,
+    should_warn,
+    recwarn,
+):
+    result_resources, was_adjusted = validate_and_adjust_resource_ratio(
+        udf_resources, max_memory_cpu_ratio, adjust
+    )
+    assert result_resources == expected_resources
+    assert was_adjusted == expected_adjusted
+    if should_warn:
+        # check warning
+        assert len(recwarn) == 1

maxframe/tests/utils.py

@@ -191,14 +191,8 @@ def assert_mf_index_dtype(idx_obj, dtype):
 
 @contextlib.contextmanager
 def create_test_volume(vol_name, oss_config):
-    test_vol_name = vol_name
     odps_entry = ODPS.from_environments()
 
-    try:
-        odps_entry.delete_volume(test_vol_name, auto_remove_dir=True, recursive=True)
-    except:
-        pass
-
     oss_test_dir_name = "test_dir_" + vol_name
     if oss_config is None:
         pytest.skip("Need oss and its config to run this test")
@@ -232,17 +226,13 @@ def create_test_volume(vol_name, oss_config):
         rolearn = oss_config.oss_rolearn
 
     oss_config.oss_bucket.put_object(oss_test_dir_name + "/", b"")
-    odps_entry.create_external_volume(
-        test_vol_name, location=test_location, rolearn=rolearn
-    )
+    odps_entry.create_external_volume(vol_name, location=test_location, rolearn=rolearn)
 
     try:
-        yield test_vol_name
+        yield vol_name
     finally:
         try:
-            odps_entry.delete_volume(
-                test_vol_name, auto_remove_dir=True, recursive=True
-            )
+            odps_entry.delete_volume(vol_name, auto_remove_dir=True, recursive=True)
         except:
             pass
 

maxframe/typing_.py

@@ -15,6 +15,7 @@
 from numbers import Integral
 from typing import List, TypeVar, Union
 
+import numpy as np
 import pandas as pd
 import pyarrow as pa
 
@@ -22,6 +23,7 @@ SlicesType = List[Union[None, Integral, slice]]
 
 TimeoutType = Union[int, float, None]
 
+PandasDType = Union[np.dtype, pd.api.extensions.ExtensionDtype]
 
 ArrowTableType = Union[pa.Table, pa.RecordBatch]
 PandasObjectTypes = Union[

maxframe/udf.py

@@ -17,6 +17,7 @@ from typing import Callable, List, Optional, Union
 
 from odps.models import Resource
 
+from .config.validators import is_positive_integer
 from .serialization import load_member
 from .serialization.serializables import (
     BoolField,
@@ -106,6 +107,7 @@ class MarkedFunction(Serializable):
     expect_resources = DictField(
         "expect_resources", FieldTypes.string, default_factory=dict
     )
+    gpu = BoolField("gpu", default=False)
 
     def __init__(self, func: Optional[Callable] = None, **kw):
         super().__init__(func=func, **kw)
@@ -177,6 +179,8 @@ def with_running_options(
     engine: Optional[str] = None,
     cpu: Optional[int] = None,
     memory: Optional[Union[str, int]] = None,
+    gu: Optional[int] = None,
+    gu_quota: Optional[Union[str, List[str]]] = None,
     **kwargs,
 ):
     """
@@ -191,6 +195,10 @@ def with_running_options(
     memory: Optional[Union[str, int]]
         The memory to run the UDF. If it is an int, it is in GB.
         If it is a str, it is in the format of "10GiB", "30MiB", etc.
+    gu: Optional[int]
+        The GU number to run the UDF.
+    gu_quota: Optional[Union[str, List[str]]]
+        The GU quota nicknames to run the UDF. The order is the priority of the usage.
     kwargs
         Other running options.
     """
@@ -204,20 +212,37 @@ def with_running_options(
             raise TypeError("memory must be an int or str")
         if isinstance(memory, int) and memory <= 0:
             raise ValueError("memory must be greater than 0")
+    if gu is not None and gu <= 0:
+        raise ValueError("gu must be greater than 0")
+    if gu is not None and (cpu or memory):
+        raise ValueError("gu can't be specified with cpu or memory")
 
     if cpu:
         resources["cpu"] = cpu
     if memory:
         resources["memory"] = memory
 
+    if isinstance(gu_quota, str):
+        gu_quota = [gu_quota]
+
+    resources["gpu"] = gu
+    resources["gu_quota"] = gu_quota
+    use_gpu = is_positive_integer(gu)
+
     def func_wrapper(func):
-        if all(v is None for v in (engine, cpu, memory)):
+        if all(v is None for v in (engine, cpu, memory, gu, gu_quota)):
             return func
         if isinstance(func, MarkedFunction):
             func.expect_engine = engine
             func.expect_resources = resources
+            func.gpu = use_gpu
             return func
-        return MarkedFunction(func, expect_engine=engine, expect_resources=resources)
+        return MarkedFunction(
+            func,
+            expect_engine=engine,
+            expect_resources=resources,
+            gpu=use_gpu,
+        )
 
     return func_wrapper
 

maxframe/utils.py

@@ -25,6 +25,7 @@ import inspect
 import io
 import itertools
 import logging
+import math
 import numbers
 import os
 import pkgutil
@@ -32,10 +33,12 @@ import random
 import re
 import struct
 import sys
+import tempfile
 import threading
 import time
 import tokenize as pytokenize
 import types
+import warnings
 import weakref
 import zlib
 from collections.abc import Hashable, Mapping
@@ -45,6 +48,7 @@ from typing import (
     Awaitable,
     Callable,
     Dict,
+    Generator,
     Iterable,
     List,
     Optional,
@@ -547,6 +551,20 @@ class ToThreadMixin:
         return self.to_thread(func, *args, wait_on_cancel=wait_on_cancel, **kwargs)
 
 
+class PatchableMixin:
+    """Patch not None field to dest_obj"""
+
+    __slots__ = ()
+
+    _patchable_attrs = tuple()
+
+    def patch_to(self, dest_obj) -> None:
+        for attr in self._patchable_attrs:
+            val = getattr(self, attr, None)
+            if val is not None:
+                setattr(dest_obj, attr, val)
+
+
 def config_odps_default_options():
     from odps import options as odps_options
 
@@ -712,7 +730,10 @@ def sbytes(x: Any) -> bytes:
     elif isinstance(x, str):
         return bytes(x, encoding="utf-8")
     else:
-        return bytes(x)
+        try:
+            return bytes(x)
+        except TypeError:
+            return bytes(str(x), encoding="utf-8")
 
 
 def is_full_slice(slc: Any) -> bool:
@@ -914,7 +935,7 @@ def stringify_path(path: Union[str, os.PathLike]) -> str:
         raise TypeError("not a path-like object")
 
 
-_memory_size_indices = {"": 0, "k": 1, "m": 2, "g": 3, "t": 4}
+_memory_size_indices = {"": 0, "b": 0, "k": 1, "m": 2, "g": 3, "t": 4}
 
 _size_pattern = re.compile(r"^([0-9.-]+)\s*([a-z]*)$")
 
@@ -1050,13 +1071,19 @@ def remove_suffix(value: str, suffix: str) -> Tuple[str, bool]:
         return value, match
 
 
-def find_objects(nested: Union[List, Dict], types: Union[Type, Tuple[Type]]) -> List:
+def find_objects(
+    nested: Union[List, Dict],
+    types: Union[None, Type, Tuple[Type]] = None,
+    checker: Callable[..., bool] = None,
+) -> List:
     found = []
     stack = [nested]
 
     while len(stack) > 0:
         it = stack.pop()
-        if isinstance(it, types):
+        if (types is not None and isinstance(it, types)) or (
+            checker is not None and checker(it)
+        ):
             found.append(it)
             continue
 
@@ -1184,7 +1211,7 @@ def arrow_type_from_str(type_str: str) -> pa.DataType:
     token_iter = pytokenize.tokenize(io.BytesIO(type_str.encode()).readline)
     value_stack, op_stack = [], []
 
-    def _pop_make_type(with_args: bool = False, combined: bool = True) -> None:
+    def _pop_make_type(with_args: bool = False, combined: bool = True):
         """
         Pops tops of value stacks, creates a DataType instance and push back
 
@@ -1208,6 +1235,23 @@ def arrow_type_from_str(type_str: str) -> pa.DataType:
         else:  # pragma: no cover
             value_stack.append(type_name)
 
+    def _pop_make_struct_field():
+        """parameterized sub-types need to be represented as tuples"""
+        nonlocal value_stack
+
+        op_stack.pop(-1)
+        if isinstance(value_stack[-1], str) and value_stack[-1].lower() in (
+            "null",
+            "not null",
+        ):
+            values = value_stack[-3:]
+            value_stack = value_stack[:-3]
+            values[-1] = values[-1] == "null"
+        else:
+            values = value_stack[-2:]
+            value_stack = value_stack[:-2]
+        value_stack.append(tuple(values))
+
     for token in token_iter:
         if token.type == pytokenize.OP:
             if token.string == ":":
@@ -1216,13 +1260,9 @@ def arrow_type_from_str(type_str: str) -> pa.DataType:
                 # gather previous sub-types
                 if op_stack[-1] in ("<", ":"):
                     _pop_make_type()
-
                 if op_stack[-1] == ":":
-                    # parameterized sub-types need to be represented as tuples
-                    op_stack.pop(-1)
-                    values = value_stack[-2:]
-                    value_stack = value_stack[:-2]
-                    value_stack.append(tuple(values))
+                    _pop_make_struct_field()
+
                 # put generated item into the parameter list
                 val = value_stack.pop(-1)
                 value_stack[-1].append(val)
@@ -1239,22 +1279,20 @@ def arrow_type_from_str(type_str: str) -> pa.DataType:
                 op_stack.pop(-1)
             elif token.string == ">":
                 _pop_make_type()
-
                 if op_stack[-1] == ":":
-                    # parameterized sub-types need to be represented as tuples
-                    op_stack.pop(-1)
-                    values = value_stack[-2:]
-                    value_stack = value_stack[:-2]
-                    value_stack.append(tuple(values))
+                    _pop_make_struct_field()
 
                 # put generated item into the parameter list
                 val = value_stack.pop(-1)
                 value_stack[-1].append(val)
                 # make DataType (i.e., list / map / struct) given args
-                _pop_make_type(True)
+                _pop_make_type(with_args=True)
                 op_stack.pop(-1)
         elif token.type == pytokenize.NAME:
-            value_stack.append(token.string)
+            if value_stack and value_stack[-1] == "not":
+                value_stack[-1] += " " + token.string
+            else:
+                value_stack.append(token.string)
         elif token.type == pytokenize.NUMBER:
             value_stack.append(int(token.string))
         elif token.type == pytokenize.ENDMARKER:
@@ -1545,3 +1583,139 @@ def cache_tileables(*tileables):
     for t in tileables:
         if isinstance(t, ENTITY_TYPE):
             t.cache = True
+
+
+def ignore_warning(func: Callable):
+    @functools.wraps(func)
+    def inner(*args, **kwargs):
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")
+            return func(*args, **kwargs)
+
+    return inner
+
+
+class ServiceLoggerAdapter(logging.LoggerAdapter):
+    extra_key_mapping = {}
+
+    def process(self, msg, kwargs):
+        merged_extra = (self.extra or {}).copy()
+        merged_extra.update(kwargs)
+
+        prefix = " ".join(
+            f"{self.extra_key_mapping.get(k) or k.capitalize()}={merged_extra[k]}"
+            for k in merged_extra.keys()
+        )
+        msg = f"[{prefix}] {msg}"
+        return msg, kwargs
+
+
+@contextmanager
+def atomic_writer(filename, mode="w", **kwargs):
+    """
+    Write to a file in an atomic way.
+    """
+    temp_fd, temp_path = tempfile.mkstemp(dir=os.path.dirname(filename) or ".")
+    os.chmod(temp_path, 0o644)
+    os.close(temp_fd)  # Close the file descriptor immediately and we reopen this later.
+
+    try:
+        # Write to temp file.
+        with open(temp_path, mode, **kwargs) as temp_file:
+            yield temp_file
+
+        # Replace the original file with the temp file atomically.
+        os.replace(temp_path, filename)
+    finally:
+        try:
+            os.remove(temp_path)
+        except OSError:
+            pass
+
+
+def prevent_called_from_pandas(level=2):
+    """Prevent method from being called from pandas"""
+    frame = sys._getframe(level)
+    called_frame = sys._getframe(1)
+    pd_pack_location = os.path.dirname(pd.__file__)
+    if frame.f_code.co_filename.startswith(pd_pack_location):
+        raise AttributeError(called_frame.f_code.co_name)
+
+
+def combine_error_message_and_traceback(
+    messages: List[str], tracebacks: List[List[str]]
+) -> str:
+    tbs = []
+    for msg, tb in zip(messages, tracebacks):
+        tbs.append("".join([msg + "\n"] + tb))
+    return "\nCaused by:\n".join(tbs)
+
+
+def generate_unique_id(byte_len: int) -> Generator[str, None, None]:
+    """
+    The ids are ensured to be unique in one generator.
+    DO NOT use this generator in global scope or singleton class members,
+    as it may not free the set.
+    """
+    generated_ids = set()
+    while True:
+        new_id = new_random_id(byte_len).hex()
+        if new_id not in generated_ids:
+            generated_ids.add(new_id)
+            yield new_id
+
+
+def validate_and_adjust_resource_ratio(
+    expect_resources: Dict[str, Any],
+    max_memory_cpu_ratio: float = None,
+    adjust: bool = False,
+) -> Tuple[Dict[str, Any], bool]:
+    """
+    Validate and optionally adjust CPU:memory ratio to meet maximum requirements.
+
+    Args:
+        expect_resources: Dictionary containing resource specifications
+        max_memory_cpu_ratio: Maximum memory/cpu ratio (if None, will use config value)
+        adjust: Whether to automatically adjust resources to meet ratio
+
+    Returns:
+        Tuple of (adjusted_resources, was_adjusted)
+    """
+    cpu = expect_resources.get("cpu") or 1
+    memory = expect_resources.get("memory")
+
+    if cpu is None or memory is None or max_memory_cpu_ratio is None:
+        return expect_resources, False
+
+    # Convert memory to GiB if it's a string
+    cpu = max(cpu, 1)
+    memory_gib = parse_size_to_megabytes(memory, default_number_unit="GiB") / 1024
+    current_ratio = memory_gib / cpu
+
+    if current_ratio > max_memory_cpu_ratio:
+        # Adjust CPU to meet maximum ratio, don't reduce resources
+        recommended_cpu = math.ceil(memory_gib / max_memory_cpu_ratio)
+        new_ratio = memory_gib / recommended_cpu
+        if adjust:
+            adjusted_resources = expect_resources.copy()
+            adjusted_resources["cpu"] = recommended_cpu
+
+            warnings.warn(
+                f"UDF resource auto-adjustment: Current UDF settings"
+                f" (CPU: {cpu}, Memory: {memory_gib}Gib, Ratio: {current_ratio:.2f})"
+                f" exceed maximum allowed ratio {max_memory_cpu_ratio:.1f}. "
+                f"Automatically adjusted to (CPU: {recommended_cpu},"
+                f" Memory: {memory_gib:.2f}:1Gib,"
+                f" Ratio: {new_ratio:.2f}:1) to meet requirements."
+            )
+            return adjusted_resources, True
+        else:
+            warnings.warn(
+                f"UDF resource ratio warning: Current UDF settings"
+                f" (CPU: {cpu}, Memory: {memory_gib}Gib, Ratio: {current_ratio:.2f})"
+                f" exceed maximum allowed ratio {max_memory_cpu_ratio:.1f}. "
+                f"Consider adjusting CPU to at least {recommended_cpu}"
+                f" (which would result in Ratio: {new_ratio:.2f}) to meet requirements."
+            )
+
+    return expect_resources, False

{maxframe-2.0.0b1.dist-info → maxframe-2.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maxframe
-Version: 2.0.0b1
+Version: 2.2.0
 Summary: MaxFrame operator-based data analyze framework
 Requires-Dist: numpy <2.0.0,>=1.19.0
 Requires-Dist: pandas >=1.0.0
@@ -19,6 +19,7 @@ Requires-Dist: pickle5 ; python_version < "3.8"
 Provides-Extra: dev
 Requires-Dist: black >=22.3.0 ; extra == 'dev'
 Requires-Dist: flake8 >=5.0.4 ; extra == 'dev'
+Requires-Dist: flake8-type-checking >=1.0.3 ; extra == 'dev'
 Requires-Dist: pre-commit >=2.15.0 ; extra == 'dev'
 Requires-Dist: graphviz >=0.20.1 ; extra == 'dev'
 Provides-Extra: test
@@ -30,7 +31,7 @@ Requires-Dist: pytest-timeout >=2.1.0 ; extra == 'test'
 Requires-Dist: matplotlib >=2.0.0 ; extra == 'test'
 Requires-Dist: lightgbm <4.0.0,>=3.0.0 ; extra == 'test'
 Requires-Dist: scikit-learn >=1.0 ; extra == 'test'
-Requires-Dist: xgboost <3.0.0,>=1.4.0 ; extra == 'test'
+Requires-Dist: xgboost <2.1.0,>=1.6.2 ; extra == 'test'
 
 MaxCompute MaxFrame Client
 ==========================