maxframe 0.1.0b4__cp311-cp311-macosx_10_9_universal2.whl → 1.0.0__cp311-cp311-macosx_10_9_universal2.whl

maxframe/tensor/utils.py

@@ -19,18 +19,13 @@ import itertools
 import operator
 from collections import OrderedDict
 from collections.abc import Iterable
-from functools import lru_cache, wraps
+from functools import wraps
 from math import ceil
 from numbers import Integral
 from typing import Dict, List, Union
 
 import numpy as np
 
-try:
-    import tiledb
-except (ImportError, OSError):  # pragma: no cover
-    tildb = None
-
 from ..core import ExecutableTuple
 from ..lib.mmh3 import hash_from_buffer
 from ..utils import lazy_import
@@ -508,7 +503,7 @@ def decide_unify_split(*splits):
 
 
 def check_out_param(out, t, casting):
-    from .base import broadcast_to
+    from .misc import broadcast_to
 
     if not hasattr(out, "shape"):
         raise TypeError("return arrays must be a tensor")
@@ -563,21 +558,6 @@ def filter_inputs(inputs):
     return [inp for inp in inputs if isinstance(inp, ENTITY_TYPE)]
 
 
-# As TileDB Ctx's creation is a bit time-consuming,
-# we just cache the Ctx
-# also remember the arguments should be hashable
-@lru_cache(10)
-def _create_tiledb_ctx(conf_tuple):
-    if conf_tuple is not None:
-        return tiledb.Ctx(dict(conf_tuple))
-    return tiledb.Ctx()
-
-
-def get_tiledb_ctx(conf):
-    key = tuple(conf.items()) if conf is not None else None
-    return _create_tiledb_ctx(key)
-
-
 # this function is only used for pandas' compatibility
 def to_numpy(pdf):
     try:

maxframe/tests/test_protocol.py

@@ -85,6 +85,40 @@ def test_error_info_json_serialize():
         deserial_err_info.reraise()
 
 
+class CannotPickleException(Exception):
+    def __reduce__(self):
+        raise ValueError
+
+
+class CannotUnpickleException(Exception):
+    @classmethod
+    def load_from_pk(cls, _):
+        raise ValueError
+
+    def __reduce__(self):
+        return type(self).load_from_pk, (0,)
+
+
+def test_error_info_fallback_json_serialize():
+    try:
+        raise CannotPickleException
+    except CannotPickleException as ex:
+        err_info1 = ErrorInfo.from_exception(ex)
+
+    try:
+        raise CannotUnpickleException
+    except CannotUnpickleException as ex:
+        err_info2 = ErrorInfo.from_exception(ex)
+
+    for err_info in (err_info1, err_info2):
+        deserial_err_info = ErrorInfo.from_json(err_info.to_json())
+        assert deserial_err_info.raw_error_source is None
+        assert deserial_err_info.raw_error_data is None
+
+        with pytest.raises(RemoteException):
+            deserial_err_info.reraise()
+
+
 def test_dag_info_json_serialize():
     try:
         raise ValueError("ERR_DATA")

maxframe/tests/test_utils.py

@@ -288,15 +288,6 @@ def test_estimate_pandas_size():
     df2 = pd.DataFrame(np.random.rand(1000, 10))
     assert utils.estimate_pandas_size(df2) == sys.getsizeof(df2)
 
-    df3 = pd.DataFrame(
-        {
-            "A": np.random.choice(["abcd", "def", "gh"], size=(1000,)),
-            "B": np.random.rand(1000),
-            "C": np.random.rand(1000),
-        }
-    )
-    assert utils.estimate_pandas_size(df3) != sys.getsizeof(df3)
-
     s1 = pd.Series(np.random.rand(1000))
     assert utils.estimate_pandas_size(s1) == sys.getsizeof(s1)
 
@@ -307,7 +298,6 @@ def test_estimate_pandas_size():
     assert utils.estimate_pandas_size(s2) == sys.getsizeof(s2)
 
     s3 = pd.Series(np.random.choice(["abcd", "def", "gh"], size=(1000,)))
-    assert utils.estimate_pandas_size(s3) != sys.getsizeof(s3)
     assert (
         pytest.approx(utils.estimate_pandas_size(s3) / sys.getsizeof(s3), abs=0.5) == 1
     )
@@ -318,7 +308,6 @@ def test_estimate_pandas_size():
     assert utils.estimate_pandas_size(idx1) == sys.getsizeof(idx1)
 
     string_idx = pd.Index(np.random.choice(["a", "bb", "cc"], size=(1000,)))
-    assert utils.estimate_pandas_size(string_idx) != sys.getsizeof(string_idx)
     assert (
         pytest.approx(
             utils.estimate_pandas_size(string_idx) / sys.getsizeof(string_idx), abs=0.5
@@ -338,7 +327,6 @@ def test_estimate_pandas_size():
         },
         index=idx2,
     )
-    assert utils.estimate_pandas_size(df4) != sys.getsizeof(df4)
     assert (
         pytest.approx(utils.estimate_pandas_size(df4) / sys.getsizeof(df4), abs=0.5)
         == 1

maxframe/tests/utils.py

@@ -14,6 +14,7 @@
 
 import asyncio
 import functools
+import hashlib
 import os
 import queue
 import socket
@@ -25,7 +26,7 @@ import pytest
 from tornado import netutil
 
 from ..core import Tileable, TileableGraph
-from ..utils import lazy_import
+from ..utils import create_sync_primitive, lazy_import, to_binary
 
 try:
     from flaky import flaky
@@ -102,7 +103,7 @@ def run_app_in_thread(app_func):
     def fixture_func(*args, **kwargs):
         app_loop = asyncio.new_event_loop()
         q = queue.Queue()
-        exit_event = asyncio.Event(loop=app_loop)
+        exit_event = create_sync_primitive(asyncio.Event, app_loop)
         app_thread = Thread(
             name="TestAppThread",
             target=app_thread_func,
@@ -162,3 +163,17 @@ def require_hadoop(func):
         not os.environ.get("WITH_HADOOP"), reason="Only run when hadoop is installed"
     )(func)
     return func
+
+
+def get_test_unique_name(size=None):
+    test_name = os.getenv("PYTEST_CURRENT_TEST", "pyodps_test")
+    digest = hashlib.md5(to_binary(test_name)).hexdigest()
+    if size:
+        digest = digest[:size]
+    return digest + "_" + str(os.getpid())
+
+
+def assert_mf_index_dtype(idx_obj, dtype):
+    from ..dataframe.core import IndexValue
+
+    assert isinstance(idx_obj, IndexValue.IndexBase) and idx_obj.dtype == dtype

maxframe/typing_.py

@@ -12,11 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import TypeVar, Union
+from numbers import Integral
+from typing import List, TypeVar, Union
 
 import pandas as pd
 import pyarrow as pa
 
+SlicesType = List[Union[None, Integral, slice]]
+
 TimeoutType = Union[int, float, None]
 
 

maxframe/udf.py

@@ -12,21 +12,48 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import shlex
 from typing import Callable, List, Optional, Union
 
 from odps.models import Resource
 
 from .serialization.serializables import (
+    BoolField,
     FieldTypes,
     FunctionField,
     ListField,
     Serializable,
+    StringField,
 )
+from .utils import tokenize
+
+
+class PythonPackOptions(Serializable):
+    _key_args = ("force_rebuild", "prefer_binary", "pre_release", "no_audit_wheel")
+
+    key = StringField("key")
+    requirements = ListField("requirements", FieldTypes.string, default_factory=list)
+    force_rebuild = BoolField("force_rebuild", default=False)
+    prefer_binary = BoolField("prefer_binary", default=False)
+    pre_release = BoolField("pre_release", default=False)
+    pack_instance_id = StringField("pack_instance_id", default=None)
+    no_audit_wheel = BoolField("no_audit_wheel", default=False)
+
+    def __init__(self, key: str = None, **kw):
+        super().__init__(key=key, **kw)
+        if self.key is None:
+            args = {k: getattr(self, k) for k in self._key_args}
+            self.key = tokenize(set(self.requirements), args)
+
+    def __repr__(self):
+        args_str = " ".join(f"{k}={getattr(self, k)}" for k in self._key_args)
+        return f"<PythonPackOptions {self.requirements} {args_str}>"
 
 
 class MarkedFunction(Serializable):
     func = FunctionField("func")
     resources = ListField("resources", FieldTypes.string, default_factory=list)
+    pythonpacks = ListField("pythonpacks", FieldTypes.reference, default_factory=list)
 
     def __init__(self, func: Optional[Callable] = None, **kw):
         super().__init__(func=func, **kw)
@@ -54,13 +81,41 @@ def with_resources(*resources: Union[str, Resource], use_wrapper_class: bool = T
     def func_wrapper(func):
         str_resources = [res_to_str(r) for r in resources]
         if not use_wrapper_class:
-            func.resources = str_resources
+            existing = getattr(func, "resources") or []
+            func.resources = existing + str_resources
+            return func
+
+        if isinstance(func, MarkedFunction):
+            func.resources = func.resources + str_resources
             return func
+        return MarkedFunction(func, resources=str_resources)
+
+    return func_wrapper
+
 
+def with_python_requirements(
+    *requirements: str,
+    force_rebuild: bool = False,
+    prefer_binary: bool = False,
+    pre_release: bool = False,
+    no_audit_wheel: bool = False,
+):
+    result_req = []
+    for req in requirements:
+        result_req.extend(shlex.split(req))
+
+    def func_wrapper(func):
+        pack_item = PythonPackOptions(
+            requirements=requirements,
+            force_rebuild=force_rebuild,
+            prefer_binary=prefer_binary,
+            pre_release=pre_release,
+            no_audit_wheel=no_audit_wheel,
+        )
         if isinstance(func, MarkedFunction):
-            func.resources = str_resources
+            func.pythonpacks.append(pack_item)
             return func
-        return MarkedFunction(func, resources=list(str_resources))
+        return MarkedFunction(func, pythonpacks=[pack_item])
 
     return func_wrapper
 
@@ -72,3 +127,7 @@ def get_udf_resources(
     func: Callable,
 ) -> List[Union[Resource, str]]:
     return getattr(func, "resources", None) or []
+
+
+def get_udf_pythonpacks(func: Callable) -> List[PythonPackOptions]:
+    return getattr(func, "pythonpacks", None) or []

maxframe/utils.py

@@ -19,7 +19,6 @@ import dataclasses
 import datetime
 import enum
 import functools
-import hashlib
 import importlib
 import inspect
 import io
@@ -33,7 +32,6 @@ import sys
 import threading
 import time
 import tokenize as pytokenize
-import traceback
 import types
 import weakref
 import zlib
@@ -76,7 +74,7 @@ from ._utils import (  # noqa: F401 # pylint: disable=unused-import
     tokenize_int,
 )
 from .lib.version import parse as parse_version
-from .typing_ import ChunkType, EntityType, TileableType, TimeoutType
+from .typing_ import TileableType, TimeoutType
 
 # make flake8 happy by referencing these imports
 NamedType = NamedType
@@ -246,58 +244,6 @@ def copy_tileables(tileables: List[TileableType], **kwargs):
     return op.new_tileables(inputs, kws=kws, output_limit=len(kws))
 
 
-def build_fetch_chunk(chunk: ChunkType, **kwargs) -> ChunkType:
-    from .core.operator import ShuffleProxy
-
-    chunk_op = chunk.op
-    params = chunk.params.copy()
-    assert not isinstance(chunk_op, ShuffleProxy)
-    # for non-shuffle nodes, we build Fetch chunks
-    # to replace original chunk
-    op = chunk_op.get_fetch_op_cls(chunk)(sparse=chunk.op.sparse, gpu=chunk.op.gpu)
-    return op.new_chunk(
-        None,
-        is_broadcaster=chunk.is_broadcaster,
-        kws=[params],
-        _key=chunk.key,
-        **kwargs,
-    )
-
-
-def build_fetch_tileable(tileable: TileableType) -> TileableType:
-    if tileable.is_coarse():
-        chunks = None
-    else:
-        chunks = []
-        for c in tileable.chunks:
-            fetch_chunk = build_fetch_chunk(c, index=c.index)
-            chunks.append(fetch_chunk)
-
-    tileable_op = tileable.op
-    params = tileable.params.copy()
-
-    new_op = tileable_op.get_fetch_op_cls(tileable)(_id=tileable_op.id)
-    return new_op.new_tileables(
-        None,
-        chunks=chunks,
-        nsplits=tileable.nsplits,
-        _key=tileable.key,
-        _id=tileable.id,
-        **params,
-    )[0]
-
-
-def build_fetch(entity: EntityType) -> EntityType:
-    from .core import CHUNK_TYPE, ENTITY_TYPE
-
-    if isinstance(entity, CHUNK_TYPE):
-        return build_fetch_chunk(entity)
-    elif isinstance(entity, ENTITY_TYPE):
-        return build_fetch_tileable(entity)
-    else:
-        raise TypeError(f"Type {type(entity)} not supported")
-
-
 def get_dtype(dtype: Union[np.dtype, pd.api.extensions.ExtensionDtype]):
     if pd.api.types.is_extension_array_dtype(dtype):
         return dtype
@@ -387,25 +333,7 @@ def build_temp_intermediate_table_name(session_id: str, tileable_key: str) -> st
 
 
 def build_session_volume_name(session_id: str) -> str:
-    return f"mf_vol_{session_id}"
-
-
-def build_tileable_dir_name(tileable_key: str) -> str:
-    m = hashlib.md5()
-    m.update(f"mf_dir_{tileable_key}".encode())
-    return m.hexdigest()
-
-
-def extract_messages_and_stacks(exc: Exception) -> Tuple[List[str], List[str]]:
-    cur_exc = exc
-    messages, stacks = [], []
-    while True:
-        messages.append(str(cur_exc))
-        stacks.append("".join(traceback.format_tb(cur_exc.__traceback__)))
-        if exc.__cause__ is None:
-            break
-        cur_exc = exc.__cause__
-    return messages, stacks
+    return f"mf_vol_{session_id.replace('-', '_')}"
 
 
 async def wait_http_response(
@@ -442,11 +370,27 @@ def format_timeout_params(timeout: TimeoutType) -> str:
         return f"?wait=1&timeout={timeout}"
 
 
-async def to_thread_pool(func, *args, pool=None, **kwargs):
-    loop = asyncio.events.get_running_loop()
-    ctx = contextvars.copy_context()
-    func_call = functools.partial(ctx.run, func, *args, **kwargs)
-    return await loop.run_in_executor(pool, func_call)
+_PrimitiveType = TypeVar("_PrimitiveType")
+
+
+def create_sync_primitive(
+    cls: Type[_PrimitiveType], loop: asyncio.AbstractEventLoop
+) -> _PrimitiveType:
+    """
+    Create an asyncio sync primitive (locks, events, etc.)
+    in a certain event loop.
+    """
+    if sys.version_info[1] < 10:
+        return cls(loop=loop)
+
+    # From Python3.10 the loop parameter has been removed. We should work around here.
+    old_loop = asyncio.get_event_loop()
+    try:
+        asyncio.set_event_loop(loop)
+        primitive = cls()
+    finally:
+        asyncio.set_event_loop(old_loop)
+    return primitive
 
 
 class ToThreadCancelledError(asyncio.CancelledError):
@@ -487,15 +431,22 @@ class ToThreadMixin:
                 thread_name_prefix=f"{type(self).__name__}Pool-{self._counter()}",
             )
 
-        task = asyncio.create_task(
-            to_thread_pool(func, *args, **kwargs, pool=self._pool)
-        )
+        loop = asyncio.events.get_running_loop()
+        ctx = contextvars.copy_context()
+        func_call = functools.partial(ctx.run, func, *args, **kwargs)
+        fut = loop.run_in_executor(self._pool, func_call)
+
         try:
-            return await asyncio.wait_for(asyncio.shield(task), timeout)
+            coro = fut
+            if wait_on_cancel:
+                coro = asyncio.shield(coro)
+            if timeout is not None:
+                coro = asyncio.wait_for(coro, timeout)
+            return await coro
         except (asyncio.CancelledError, asyncio.TimeoutError) as ex:
             if not wait_on_cancel:
                 raise
-            result = await task
+            result = await fut
             raise ToThreadCancelledError(*ex.args, result=result)
 
     def ensure_async_call(
@@ -519,6 +470,7 @@ def config_odps_default_options():
         "metaservice.client.cache.enable": "false",
         "odps.sql.session.result.cache.enable": "false",
         "odps.sql.submit.mode": "script",
+        "odps.sql.job.max.time.hours": 72,
     }
 
 
@@ -883,8 +835,41 @@ def parse_readable_size(value: Union[str, int, float]) -> Tuple[float, bool]:
         raise ValueError(f"Unknown limitation value: {value}")
 
 
-def remove_suffix(value: str, suffix: str) -> str:
-    return value[: -len(suffix)] if value.endswith(suffix) else value
+def remove_suffix(value: str, suffix: str) -> Tuple[str, bool]:
+    """
+    Remove a suffix from a given string if it exists.
+
+    Parameters
+    ----------
+    value : str
+        The original string.
+    suffix : str
+        The suffix to be removed.
+
+    Returns
+    -------
+    Tuple[str, bool]
+        A tuple containing the modified string and a boolean indicating whether the suffix was found.
+    """
+
+    # Check if the suffix is an empty string
+    if len(suffix) == 0:
+        # If the suffix is empty, return the original string with True
+        return value, True
+
+    # Check if the length of the value is less than the length of the suffix
+    if len(value) < len(suffix):
+        # If the value is shorter than the suffix, it cannot have the suffix
+        return value, False
+
+    # Check if the suffix matches the end of the value
+    match = value.endswith(suffix)
+
+    # If the suffix is found, remove it; otherwise, return the original string
+    if match:
+        return value[: -len(suffix)], match
+    else:
+        return value, match
 
 
 def find_objects(nested: Union[List, Dict], types: Union[Type, Tuple[Type]]) -> List:
@@ -1106,3 +1091,44 @@ def get_python_tag():
     # todo add implementation suffix for non-GIL tags when PEP703 is ready
     version_info = sys.version_info
     return f"cp{version_info[0]}{version_info[1]}"
+
+
+def get_item_if_scalar(val: Any) -> Any:
+    if isinstance(val, np.ndarray) and val.shape == ():
+        return val.item()
+    return val
+
+
+def collect_leaf_operators(root) -> List[Type]:
+    result = []
+
+    def _collect(op_type):
+        if len(op_type.__subclasses__()) == 0:
+            result.append(op_type)
+        for subclass in op_type.__subclasses__():
+            _collect(subclass)
+
+    _collect(root)
+    return result
+
+
+@contextmanager
+def sync_pyodps_options():
+    from odps.config import OptionError
+    from odps.config import option_context as pyodps_option_context
+
+    from .config import options
+
+    with pyodps_option_context() as cfg:
+        cfg.local_timezone = options.local_timezone
+        if options.session.enable_schema:
+            try:
+                cfg.enable_schema = options.session.enable_schema
+            except (AttributeError, OptionError):
+                # fixme enable_schema only supported in PyODPS 0.12.0 or later
+                cfg.always_enable_schema = options.session.enable_schema
+        yield
+
+
+def str_to_bool(s: Optional[str]) -> Optional[bool]:
+    return s.lower().strip() in ("true", "1") if s is not None else None

{maxframe-0.1.0b4.dist-info → maxframe-1.0.0.dist-info}/METADATA

@@ -1,33 +1,33 @@
 Metadata-Version: 2.1
 Name: maxframe
-Version: 0.1.0b4
+Version: 1.0.0
 Summary: MaxFrame operator-based data analyze framework
-Requires-Dist: numpy >=1.19.0
-Requires-Dist: pandas >=1.0.0
-Requires-Dist: pyodps >=0.11.5
-Requires-Dist: scipy >=1.0
-Requires-Dist: pyarrow >=1.0.0
-Requires-Dist: msgpack >=1.0.0
-Requires-Dist: traitlets >=5.0
-Requires-Dist: cloudpickle >=1.5.0
-Requires-Dist: pyyaml >=5.1
-Requires-Dist: tornado >=6.0
-Requires-Dist: defusedxml >=0.5.0
-Requires-Dist: tqdm >=4.1.0
-Requires-Dist: importlib-metadata >=1.4
-Requires-Dist: pickle5 ; python_version < "3.8"
+Requires-Dist: numpy<2.0.0,>=1.19.0
+Requires-Dist: pandas>=1.0.0
+Requires-Dist: pyodps>=0.11.6.1
+Requires-Dist: scipy>=1.0
+Requires-Dist: pyarrow>=1.0.0
+Requires-Dist: msgpack>=1.0.0
+Requires-Dist: traitlets>=5.0
+Requires-Dist: cloudpickle<3.0.0,>=1.5.0
+Requires-Dist: pyyaml>=5.1
+Requires-Dist: tornado>=6.0
+Requires-Dist: defusedxml>=0.5.0
+Requires-Dist: tqdm>=4.1.0
+Requires-Dist: importlib-metadata>=1.4
+Requires-Dist: pickle5; python_version < "3.8"
 Provides-Extra: dev
-Requires-Dist: black >=22.3.0 ; extra == 'dev'
-Requires-Dist: flake8 >=5.0.4 ; extra == 'dev'
-Requires-Dist: pre-commit >=2.15.0 ; extra == 'dev'
-Requires-Dist: graphviz >=0.20.1 ; extra == 'dev'
+Requires-Dist: black>=22.3.0; extra == "dev"
+Requires-Dist: flake8>=5.0.4; extra == "dev"
+Requires-Dist: pre-commit>=2.15.0; extra == "dev"
+Requires-Dist: graphviz>=0.20.1; extra == "dev"
 Provides-Extra: test
-Requires-Dist: mock ; extra == 'test'
-Requires-Dist: pytest >=7.3.1 ; extra == 'test'
-Requires-Dist: pytest-cov >=4.1.0 ; extra == 'test'
-Requires-Dist: pytest-asyncio >=0.21.0 ; extra == 'test'
-Requires-Dist: pytest-timeout >=2.1.0 ; extra == 'test'
-Requires-Dist: matplotlib >=2.0.0 ; extra == 'test'
+Requires-Dist: mock; extra == "test"
+Requires-Dist: pytest>=7.3.1; extra == "test"
+Requires-Dist: pytest-cov>=4.1.0; extra == "test"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "test"
+Requires-Dist: pytest-timeout>=2.1.0; extra == "test"
+Requires-Dist: matplotlib>=2.0.0; extra == "test"
 
 MaxCompute MaxFrame Client
 ==========================