maxframe 0.1.0b5__cp39-cp39-macosx_10_9_universal2.whl → 1.0.0__cp39-cp39-macosx_10_9_universal2.whl

maxframe/tests/utils.py

@@ -14,6 +14,7 @@
 
 import asyncio
 import functools
+import hashlib
 import os
 import queue
 import socket
@@ -25,7 +26,7 @@ import pytest
 from tornado import netutil
 
 from ..core import Tileable, TileableGraph
-from ..utils import lazy_import
+from ..utils import create_sync_primitive, lazy_import, to_binary
 
 try:
     from flaky import flaky
@@ -102,7 +103,7 @@ def run_app_in_thread(app_func):
     def fixture_func(*args, **kwargs):
         app_loop = asyncio.new_event_loop()
         q = queue.Queue()
-        exit_event = asyncio.Event(loop=app_loop)
+        exit_event = create_sync_primitive(asyncio.Event, app_loop)
         app_thread = Thread(
             name="TestAppThread",
             target=app_thread_func,
@@ -162,3 +163,17 @@ def require_hadoop(func):
         not os.environ.get("WITH_HADOOP"), reason="Only run when hadoop is installed"
     )(func)
     return func
+
+
+def get_test_unique_name(size=None):
+    test_name = os.getenv("PYTEST_CURRENT_TEST", "pyodps_test")
+    digest = hashlib.md5(to_binary(test_name)).hexdigest()
+    if size:
+        digest = digest[:size]
+    return digest + "_" + str(os.getpid())
+
+
+def assert_mf_index_dtype(idx_obj, dtype):
+    from ..dataframe.core import IndexValue
+
+    assert isinstance(idx_obj, IndexValue.IndexBase) and idx_obj.dtype == dtype

maxframe/typing_.py

@@ -12,11 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import TypeVar, Union
+from numbers import Integral
+from typing import List, TypeVar, Union
 
 import pandas as pd
 import pyarrow as pa
 
+SlicesType = List[Union[None, Integral, slice]]
+
 TimeoutType = Union[int, float, None]
 
 

maxframe/udf.py

@@ -29,28 +29,25 @@ from .utils import tokenize
 
 
 class PythonPackOptions(Serializable):
+    _key_args = ("force_rebuild", "prefer_binary", "pre_release", "no_audit_wheel")
+
     key = StringField("key")
     requirements = ListField("requirements", FieldTypes.string, default_factory=list)
     force_rebuild = BoolField("force_rebuild", default=False)
     prefer_binary = BoolField("prefer_binary", default=False)
     pre_release = BoolField("pre_release", default=False)
     pack_instance_id = StringField("pack_instance_id", default=None)
+    no_audit_wheel = BoolField("no_audit_wheel", default=False)
 
     def __init__(self, key: str = None, **kw):
         super().__init__(key=key, **kw)
         if self.key is None:
-            args = {
-                "force_rebuild": self.force_rebuild,
-                "prefer_binary": self.prefer_binary,
-                "pre_release": self.pre_release,
-            }
+            args = {k: getattr(self, k) for k in self._key_args}
             self.key = tokenize(set(self.requirements), args)
 
     def __repr__(self):
-        return (
-            f"<PythonPackOptions {self.requirements} force_rebuild={self.force_rebuild} "
-            f"prefer_binary={self.prefer_binary} pre_release={self.pre_release}>"
-        )
+        args_str = " ".join(f"{k}={getattr(self, k)}" for k in self._key_args)
+        return f"<PythonPackOptions {self.requirements} {args_str}>"
 
 
 class MarkedFunction(Serializable):
@@ -101,6 +98,7 @@ def with_python_requirements(
     force_rebuild: bool = False,
     prefer_binary: bool = False,
     pre_release: bool = False,
+    no_audit_wheel: bool = False,
 ):
     result_req = []
     for req in requirements:
@@ -112,6 +110,7 @@ def with_python_requirements(
             force_rebuild=force_rebuild,
             prefer_binary=prefer_binary,
             pre_release=pre_release,
+            no_audit_wheel=no_audit_wheel,
         )
         if isinstance(func, MarkedFunction):
             func.pythonpacks.append(pack_item)

maxframe/utils.py

@@ -19,7 +19,6 @@ import dataclasses
 import datetime
 import enum
 import functools
-import hashlib
 import importlib
 import inspect
 import io
@@ -33,7 +32,6 @@ import sys
 import threading
 import time
 import tokenize as pytokenize
-import traceback
 import types
 import weakref
 import zlib
@@ -76,7 +74,7 @@ from ._utils import (  # noqa: F401 # pylint: disable=unused-import
     tokenize_int,
 )
 from .lib.version import parse as parse_version
-from .typing_ import ChunkType, EntityType, TileableType, TimeoutType
+from .typing_ import TileableType, TimeoutType
 
 # make flake8 happy by referencing these imports
 NamedType = NamedType
@@ -246,58 +244,6 @@ def copy_tileables(tileables: List[TileableType], **kwargs):
     return op.new_tileables(inputs, kws=kws, output_limit=len(kws))
 
 
-def build_fetch_chunk(chunk: ChunkType, **kwargs) -> ChunkType:
-    from .core.operator import ShuffleProxy
-
-    chunk_op = chunk.op
-    params = chunk.params.copy()
-    assert not isinstance(chunk_op, ShuffleProxy)
-    # for non-shuffle nodes, we build Fetch chunks
-    # to replace original chunk
-    op = chunk_op.get_fetch_op_cls(chunk)(sparse=chunk.op.sparse, gpu=chunk.op.gpu)
-    return op.new_chunk(
-        None,
-        is_broadcaster=chunk.is_broadcaster,
-        kws=[params],
-        _key=chunk.key,
-        **kwargs,
-    )
-
-
-def build_fetch_tileable(tileable: TileableType) -> TileableType:
-    if tileable.is_coarse():
-        chunks = None
-    else:
-        chunks = []
-        for c in tileable.chunks:
-            fetch_chunk = build_fetch_chunk(c, index=c.index)
-            chunks.append(fetch_chunk)
-
-    tileable_op = tileable.op
-    params = tileable.params.copy()
-
-    new_op = tileable_op.get_fetch_op_cls(tileable)(_id=tileable_op.id)
-    return new_op.new_tileables(
-        None,
-        chunks=chunks,
-        nsplits=tileable.nsplits,
-        _key=tileable.key,
-        _id=tileable.id,
-        **params,
-    )[0]
-
-
-def build_fetch(entity: EntityType) -> EntityType:
-    from .core import CHUNK_TYPE, ENTITY_TYPE
-
-    if isinstance(entity, CHUNK_TYPE):
-        return build_fetch_chunk(entity)
-    elif isinstance(entity, ENTITY_TYPE):
-        return build_fetch_tileable(entity)
-    else:
-        raise TypeError(f"Type {type(entity)} not supported")
-
-
 def get_dtype(dtype: Union[np.dtype, pd.api.extensions.ExtensionDtype]):
     if pd.api.types.is_extension_array_dtype(dtype):
         return dtype
@@ -387,25 +333,7 @@ def build_temp_intermediate_table_name(session_id: str, tileable_key: str) -> st
 
 
 def build_session_volume_name(session_id: str) -> str:
-    return f"mf_vol_{session_id}"
-
-
-def build_tileable_dir_name(tileable_key: str) -> str:
-    m = hashlib.md5()
-    m.update(f"mf_dir_{tileable_key}".encode())
-    return m.hexdigest()
-
-
-def extract_messages_and_stacks(exc: Exception) -> Tuple[List[str], List[str]]:
-    cur_exc = exc
-    messages, stacks = [], []
-    while True:
-        messages.append(str(cur_exc))
-        stacks.append("".join(traceback.format_tb(cur_exc.__traceback__)))
-        if exc.__cause__ is None:
-            break
-        cur_exc = exc.__cause__
-    return messages, stacks
+    return f"mf_vol_{session_id.replace('-', '_')}"
 
 
 async def wait_http_response(
@@ -442,11 +370,27 @@ def format_timeout_params(timeout: TimeoutType) -> str:
         return f"?wait=1&timeout={timeout}"
 
 
-async def to_thread_pool(func, *args, pool=None, **kwargs):
-    loop = asyncio.events.get_running_loop()
-    ctx = contextvars.copy_context()
-    func_call = functools.partial(ctx.run, func, *args, **kwargs)
-    return await loop.run_in_executor(pool, func_call)
+_PrimitiveType = TypeVar("_PrimitiveType")
+
+
+def create_sync_primitive(
+    cls: Type[_PrimitiveType], loop: asyncio.AbstractEventLoop
+) -> _PrimitiveType:
+    """
+    Create an asyncio sync primitive (locks, events, etc.)
+    in a certain event loop.
+    """
+    if sys.version_info[1] < 10:
+        return cls(loop=loop)
+
+    # From Python3.10 the loop parameter has been removed. We should work around here.
+    old_loop = asyncio.get_event_loop()
+    try:
+        asyncio.set_event_loop(loop)
+        primitive = cls()
+    finally:
+        asyncio.set_event_loop(old_loop)
+    return primitive
 
 
 class ToThreadCancelledError(asyncio.CancelledError):
@@ -487,15 +431,22 @@ class ToThreadMixin:
                 thread_name_prefix=f"{type(self).__name__}Pool-{self._counter()}",
             )
 
-        task = asyncio.create_task(
-            to_thread_pool(func, *args, **kwargs, pool=self._pool)
-        )
+        loop = asyncio.events.get_running_loop()
+        ctx = contextvars.copy_context()
+        func_call = functools.partial(ctx.run, func, *args, **kwargs)
+        fut = loop.run_in_executor(self._pool, func_call)
+
         try:
-            return await asyncio.wait_for(asyncio.shield(task), timeout)
+            coro = fut
+            if wait_on_cancel:
+                coro = asyncio.shield(coro)
+            if timeout is not None:
+                coro = asyncio.wait_for(coro, timeout)
+            return await coro
         except (asyncio.CancelledError, asyncio.TimeoutError) as ex:
             if not wait_on_cancel:
                 raise
-            result = await task
+            result = await fut
             raise ToThreadCancelledError(*ex.args, result=result)
 
     def ensure_async_call(
@@ -519,6 +470,7 @@ def config_odps_default_options():
         "metaservice.client.cache.enable": "false",
         "odps.sql.session.result.cache.enable": "false",
         "odps.sql.submit.mode": "script",
+        "odps.sql.job.max.time.hours": 72,
     }
 
 
@@ -883,8 +835,41 @@ def parse_readable_size(value: Union[str, int, float]) -> Tuple[float, bool]:
         raise ValueError(f"Unknown limitation value: {value}")
 
 
-def remove_suffix(value: str, suffix: str) -> str:
-    return value[: -len(suffix)] if value.endswith(suffix) else value
+def remove_suffix(value: str, suffix: str) -> Tuple[str, bool]:
+    """
+    Remove a suffix from a given string if it exists.
+
+    Parameters
+    ----------
+    value : str
+        The original string.
+    suffix : str
+        The suffix to be removed.
+
+    Returns
+    -------
+    Tuple[str, bool]
+        A tuple containing the modified string and a boolean indicating whether the suffix was found.
+    """
+
+    # Check if the suffix is an empty string
+    if len(suffix) == 0:
+        # If the suffix is empty, return the original string with True
+        return value, True
+
+    # Check if the length of the value is less than the length of the suffix
+    if len(value) < len(suffix):
+        # If the value is shorter than the suffix, it cannot have the suffix
+        return value, False
+
+    # Check if the suffix matches the end of the value
+    match = value.endswith(suffix)
+
+    # If the suffix is found, remove it; otherwise, return the original string
+    if match:
+        return value[: -len(suffix)], match
+    else:
+        return value, match
 
 
 def find_objects(nested: Union[List, Dict], types: Union[Type, Tuple[Type]]) -> List:
@@ -1112,3 +1097,38 @@ def get_item_if_scalar(val: Any) -> Any:
     if isinstance(val, np.ndarray) and val.shape == ():
         return val.item()
     return val
+
+
+def collect_leaf_operators(root) -> List[Type]:
+    result = []
+
+    def _collect(op_type):
+        if len(op_type.__subclasses__()) == 0:
+            result.append(op_type)
+        for subclass in op_type.__subclasses__():
+            _collect(subclass)
+
+    _collect(root)
+    return result
+
+
+@contextmanager
+def sync_pyodps_options():
+    from odps.config import OptionError
+    from odps.config import option_context as pyodps_option_context
+
+    from .config import options
+
+    with pyodps_option_context() as cfg:
+        cfg.local_timezone = options.local_timezone
+        if options.session.enable_schema:
+            try:
+                cfg.enable_schema = options.session.enable_schema
+            except (AttributeError, OptionError):
+                # fixme enable_schema only supported in PyODPS 0.12.0 or later
+                cfg.always_enable_schema = options.session.enable_schema
+        yield
+
+
+def str_to_bool(s: Optional[str]) -> Optional[bool]:
+    return s.lower().strip() in ("true", "1") if s is not None else None

{maxframe-0.1.0b5.dist-info → maxframe-1.0.0.dist-info}/METADATA

@@ -1,33 +1,33 @@
 Metadata-Version: 2.1
 Name: maxframe
-Version: 0.1.0b5
+Version: 1.0.0
 Summary: MaxFrame operator-based data analyze framework
-Requires-Dist: numpy >=1.19.0
-Requires-Dist: pandas >=1.0.0
-Requires-Dist: pyodps >=0.11.6.1
-Requires-Dist: scipy >=1.0
-Requires-Dist: pyarrow >=1.0.0
-Requires-Dist: msgpack >=1.0.0
-Requires-Dist: traitlets >=5.0
-Requires-Dist: cloudpickle >=1.5.0
-Requires-Dist: pyyaml >=5.1
-Requires-Dist: tornado >=6.0
-Requires-Dist: defusedxml >=0.5.0
-Requires-Dist: tqdm >=4.1.0
-Requires-Dist: importlib-metadata >=1.4
-Requires-Dist: pickle5 ; python_version < "3.8"
+Requires-Dist: numpy<2.0.0,>=1.19.0
+Requires-Dist: pandas>=1.0.0
+Requires-Dist: pyodps>=0.11.6.1
+Requires-Dist: scipy>=1.0
+Requires-Dist: pyarrow>=1.0.0
+Requires-Dist: msgpack>=1.0.0
+Requires-Dist: traitlets>=5.0
+Requires-Dist: cloudpickle<3.0.0,>=1.5.0
+Requires-Dist: pyyaml>=5.1
+Requires-Dist: tornado>=6.0
+Requires-Dist: defusedxml>=0.5.0
+Requires-Dist: tqdm>=4.1.0
+Requires-Dist: importlib-metadata>=1.4
+Requires-Dist: pickle5; python_version < "3.8"
 Provides-Extra: dev
-Requires-Dist: black >=22.3.0 ; extra == 'dev'
-Requires-Dist: flake8 >=5.0.4 ; extra == 'dev'
-Requires-Dist: pre-commit >=2.15.0 ; extra == 'dev'
-Requires-Dist: graphviz >=0.20.1 ; extra == 'dev'
+Requires-Dist: black>=22.3.0; extra == "dev"
+Requires-Dist: flake8>=5.0.4; extra == "dev"
+Requires-Dist: pre-commit>=2.15.0; extra == "dev"
+Requires-Dist: graphviz>=0.20.1; extra == "dev"
 Provides-Extra: test
-Requires-Dist: mock ; extra == 'test'
-Requires-Dist: pytest >=7.3.1 ; extra == 'test'
-Requires-Dist: pytest-cov >=4.1.0 ; extra == 'test'
-Requires-Dist: pytest-asyncio >=0.21.0 ; extra == 'test'
-Requires-Dist: pytest-timeout >=2.1.0 ; extra == 'test'
-Requires-Dist: matplotlib >=2.0.0 ; extra == 'test'
+Requires-Dist: mock; extra == "test"
+Requires-Dist: pytest>=7.3.1; extra == "test"
+Requires-Dist: pytest-cov>=4.1.0; extra == "test"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "test"
+Requires-Dist: pytest-timeout>=2.1.0; extra == "test"
+Requires-Dist: matplotlib>=2.0.0; extra == "test"
 
 MaxCompute MaxFrame Client
 ==========================