maxframe 1.0.0rc1__cp311-cp311-macosx_10_9_universal2.whl → 1.0.0rc3__cp311-cp311-macosx_10_9_universal2.whl

maxframe/tensor/{base → misc}/transpose.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 1999-2024 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -66,33 +64,39 @@ class TensorTranspose(TensorHasInput, TensorOperatorMixin):
 
 def transpose(a, axes=None):
     """
-    Permute the dimensions of a tensor.
+    Returns an array with axes transposed.
+
+    For a 1-D array, this returns an unchanged view of the original array, as a
+    transposed vector is simply the same vector.
+    To convert a 1-D array into a 2-D column vector, an additional dimension
+    must be added, e.g., ``mt.atleast_2d(a).T`` achieves this, as does
+    ``a[:, mt.newaxis]``.
+    For a 2-D array, this is the standard matrix transpose.
+    For an n-D array, if axes are given, their order indicates how the
+    axes are permuted (see Examples). If axes are not provided, then
+    ``transpose(a).shape == a.shape[::-1]``.
 
     Parameters
     ----------
     a : array_like
-        Input tensor.
-    axes : list of ints, optional
-        By default, reverse the dimensions, otherwise permute the axes
-        according to the values given.
+        Input array.
+    axes : tuple or list of ints, optional
+        If specified, it must be a tuple or list which contains a permutation
+        of [0,1,...,N-1] where N is the number of axes of `a`. The `i`'th axis
+        of the returned array will correspond to the axis numbered ``axes[i]``
+        of the input. If not specified, defaults to ``range(a.ndim)[::-1]``,
+        which reverses the order of the axes.
 
     Returns
     -------
-    p : Tensor
-        `a` with its axes permuted.  A view is returned whenever
-        possible.
-
-    See Also
-    --------
-    moveaxis
-    argsort
+    p : ndarray
+        `a` with its axes permuted. A view is returned whenever possible.
 
     Notes
     -----
-    Use `transpose(a, argsort(axes))` to invert the transposition of tensors
+    Use ``transpose(a, argsort(axes))`` to invert the transposition of tensors
     when using the `axes` keyword argument.
 
-    Transposing a 1-D array returns an unchanged view of the original tensor.
 
     Examples
     --------
@@ -121,5 +125,5 @@ def transpose(a, axes=None):
         axes = list(range(a.ndim))[::-1]
     else:
         axes = list(axes)
-    op = TensorTranspose(axes, dtype=a.dtype, sparse=a.issparse())
+    op = TensorTranspose(axes)
     return op(a)

maxframe/tensor/{base → misc}/unique.py

@@ -15,7 +15,7 @@
 
 import numpy as np
 
-from ... import opcodes as OperandDef
+from ... import opcodes
 from ...serialization.serializables import BoolField, Int32Field
 from ..core import TensorOrder
 from ..operators import TensorHasInput, TensorOperatorMixin
@@ -23,7 +23,7 @@ from ..utils import validate_axis
 
 
 class TensorUnique(TensorHasInput, TensorOperatorMixin):
-    _op_type_ = OperandDef.UNIQUE
+    _op_type_ = opcodes.UNIQUE
 
     return_index = BoolField("return_index", default=False)
     return_inverse = BoolField("return_inverse", default=False)

maxframe/tensor/operators.py

@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 from ..core import OutputType
 from ..core.operator import (
-    Fuse,
     HasInput,
     MapReduceOperator,
     Operator,
@@ -115,9 +115,3 @@ class TensorMapReduceOperator(MapReduceOperator):
     _output_type_ = OutputType.tensor
 
     dtype = DataTypeField("dtype", default=None)
-
-
-class TensorFuse(Fuse):
-    _output_type_ = OutputType.tensor
-
-    dtype = DataTypeField("dtype", default=None)

maxframe/tensor/random/core.py

@@ -19,9 +19,9 @@ from contextlib import contextmanager
 import numpy as np
 
 from ...serialization.serializables import FieldTypes, Int32Field, TupleField
-from ..base import broadcast_to
 from ..core import TENSOR_TYPE
 from ..datasource import tensor as astensor
+from ..misc import broadcast_to
 from ..operators import TensorMapReduceOperator, TensorOperator, TensorOperatorMixin
 from ..utils import broadcast_shape
 

maxframe/tensor/reduction/count_nonzero.py

@@ -22,6 +22,7 @@ from .core import TensorReduction, TensorReductionMixin
 
 class TensorCountNonzero(TensorReduction, TensorReductionMixin):
     _op_type_ = opcodes.COUNT_NONZERO
+    _func_name = "count_nonzero"
 
     def __init__(self, dtype=None, **kw):
         if dtype is None:

maxframe/tensor/reduction/mean.py

@@ -23,6 +23,7 @@ from .core import TensorReduction, TensorReductionMixin
 
 class TensorMean(TensorReduction, TensorReductionMixin):
     _op_type_ = opcodes.MEAN
+    _func_name = "mean"
 
 
 def mean(a, axis=None, dtype=None, out=None, keepdims=None):

maxframe/tensor/reduction/nanmean.py

@@ -23,6 +23,7 @@ from .core import TensorReduction, TensorReductionMixin
 
 class TensorNanMean(TensorReduction, TensorReductionMixin):
     _op_type_ = opcodes.NANMEAN
+    _func_name = "nanmean"
 
 
 def nanmean(a, axis=None, dtype=None, out=None, keepdims=None):

maxframe/tensor/reduction/nanvar.py

@@ -24,6 +24,7 @@ from .core import TensorReduction, TensorReductionMixin
 
 class TensorNanMoment(TensorReduction, TensorReductionMixin):
     _op_type_ = opcodes.NANMOMENT
+    _func_name = "nanvar"
 
     moment = Int32Field("moment", default=2)
     ddof = Int32Field("ddof", default=None)
@@ -36,6 +37,7 @@ class TensorNanMoment(TensorReduction, TensorReductionMixin):
 
 class TensorNanVar(TensorReduction, TensorReductionMixin):
     _op_type_ = opcodes.NANVAR
+    _func_name = "nanvar"
 
     ddof = Int32Field("ddof", default=0)
 

maxframe/tensor/reduction/tests/test_reduction.py

@@ -17,8 +17,11 @@
 import numpy as np
 import pytest
 
+from maxframe.tensor.reduction.core import TensorReduction
+
+from ....utils import collect_leaf_operators
 from ...datasource import ones, tensor
-from .. import all
+from .. import *  # noqa: F401
 
 
 def test_base_reduction():
@@ -179,3 +182,11 @@ def test_var_reduction():
 
     res1 = var(ones((10, 8, 8), chunk_size=3), axis=1)
     assert res1.shape == (10, 8)
+
+
+def test_reduction_op_func_name():
+    # make sure all the binary op has defined the func name.
+
+    results = collect_leaf_operators(TensorReduction)
+    for op_type in results:
+        assert hasattr(op_type, "_func_name")

maxframe/tensor/reduction/var.py

@@ -42,6 +42,7 @@ def reduce_var_square(var_square, avg_diff, count, op, axis, sum_func):
 
 class TensorMoment(TensorReduction, TensorReductionMixin):
     _op_type_ = opcodes.MOMENT
+    _func_name = "var"
 
     moment = Int32Field("moment", default=2)
     ddof = Int32Field("ddof", default=None)
@@ -54,6 +55,7 @@ class TensorMoment(TensorReduction, TensorReductionMixin):
 
 class TensorVar(TensorReduction, TensorReductionMixin):
     _op_type_ = opcodes.VAR
+    _func_name = "var"
 
     ddof = Int32Field("ddof", default=0)
 

maxframe/tensor/statistics/quantile.py

@@ -16,7 +16,7 @@ from collections.abc import Iterable
 
 import numpy as np
 
-from ... import opcodes as OperandDef
+from ... import opcodes
 from ...core import ENTITY_TYPE
 from ...serialization.serializables import AnyField, BoolField, KeyField, StringField
 from ..core import TENSOR_TYPE, TensorOrder
@@ -43,7 +43,7 @@ q_error_msg = "Quantiles must be in the range [0, 1]"
 
 class TensorQuantile(TensorOperator, TensorOperatorMixin):
     __slots__ = ("q_error_msg",)
-    _op_type_ = OperandDef.QUANTILE
+    _op_type_ = opcodes.QUANTILE
 
     a = KeyField("a")
     q = AnyField("q")

maxframe/tensor/utils.py

@@ -19,18 +19,13 @@ import itertools
 import operator
 from collections import OrderedDict
 from collections.abc import Iterable
-from functools import lru_cache, wraps
+from functools import wraps
 from math import ceil
 from numbers import Integral
 from typing import Dict, List, Union
 
 import numpy as np
 
-try:
-    import tiledb
-except (ImportError, OSError):  # pragma: no cover
-    tildb = None
-
 from ..core import ExecutableTuple
 from ..lib.mmh3 import hash_from_buffer
 from ..utils import lazy_import
@@ -508,7 +503,7 @@ def decide_unify_split(*splits):
 
 
 def check_out_param(out, t, casting):
-    from .base import broadcast_to
+    from .misc import broadcast_to
 
     if not hasattr(out, "shape"):
         raise TypeError("return arrays must be a tensor")
@@ -563,21 +558,6 @@ def filter_inputs(inputs):
     return [inp for inp in inputs if isinstance(inp, ENTITY_TYPE)]
 
 
-# As TileDB Ctx's creation is a bit time-consuming,
-# we just cache the Ctx
-# also remember the arguments should be hashable
-@lru_cache(10)
-def _create_tiledb_ctx(conf_tuple):
-    if conf_tuple is not None:
-        return tiledb.Ctx(dict(conf_tuple))
-    return tiledb.Ctx()
-
-
-def get_tiledb_ctx(conf):
-    key = tuple(conf.items()) if conf is not None else None
-    return _create_tiledb_ctx(key)
-
-
 # this function is only used for pandas' compatibility
 def to_numpy(pdf):
     try:

maxframe/tests/utils.py

@@ -14,6 +14,7 @@
 
 import asyncio
 import functools
+import hashlib
 import os
 import queue
 import socket
@@ -25,7 +26,7 @@ import pytest
 from tornado import netutil
 
 from ..core import Tileable, TileableGraph
-from ..utils import create_event, lazy_import
+from ..utils import create_sync_primitive, lazy_import, to_binary
 
 try:
     from flaky import flaky
@@ -102,7 +103,7 @@ def run_app_in_thread(app_func):
     def fixture_func(*args, **kwargs):
         app_loop = asyncio.new_event_loop()
         q = queue.Queue()
-        exit_event = create_event(app_loop)
+        exit_event = create_sync_primitive(asyncio.Event, app_loop)
         app_thread = Thread(
             name="TestAppThread",
             target=app_thread_func,
@@ -162,3 +163,11 @@ def require_hadoop(func):
         not os.environ.get("WITH_HADOOP"), reason="Only run when hadoop is installed"
     )(func)
     return func
+
+
+def get_test_unique_name(size=None):
+    test_name = os.getenv("PYTEST_CURRENT_TEST", "pyodps_test")
+    digest = hashlib.md5(to_binary(test_name)).hexdigest()
+    if size:
+        digest = digest[:size]
+    return digest + "_" + str(os.getpid())

maxframe/typing_.py

@@ -12,11 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import TypeVar, Union
+from numbers import Integral
+from typing import List, TypeVar, Union
 
 import pandas as pd
 import pyarrow as pa
 
+SlicesType = List[Union[None, Integral, slice]]
+
 TimeoutType = Union[int, float, None]
 
 

maxframe/udf.py

@@ -29,28 +29,25 @@ from .utils import tokenize
 
 
 class PythonPackOptions(Serializable):
+    _key_args = ("force_rebuild", "prefer_binary", "pre_release", "no_audit_wheel")
+
     key = StringField("key")
     requirements = ListField("requirements", FieldTypes.string, default_factory=list)
     force_rebuild = BoolField("force_rebuild", default=False)
     prefer_binary = BoolField("prefer_binary", default=False)
     pre_release = BoolField("pre_release", default=False)
     pack_instance_id = StringField("pack_instance_id", default=None)
+    no_audit_wheel = BoolField("no_audit_wheel", default=False)
 
     def __init__(self, key: str = None, **kw):
         super().__init__(key=key, **kw)
         if self.key is None:
-            args = {
-                "force_rebuild": self.force_rebuild,
-                "prefer_binary": self.prefer_binary,
-                "pre_release": self.pre_release,
-            }
+            args = {k: getattr(self, k) for k in self._key_args}
             self.key = tokenize(set(self.requirements), args)
 
     def __repr__(self):
-        return (
-            f"<PythonPackOptions {self.requirements} force_rebuild={self.force_rebuild} "
-            f"prefer_binary={self.prefer_binary} pre_release={self.pre_release}>"
-        )
+        args_str = " ".join(f"{k}={getattr(self, k)}" for k in self._key_args)
+        return f"<PythonPackOptions {self.requirements} {args_str}>"
 
 
 class MarkedFunction(Serializable):
@@ -101,6 +98,7 @@ def with_python_requirements(
     force_rebuild: bool = False,
     prefer_binary: bool = False,
     pre_release: bool = False,
+    no_audit_wheel: bool = False,
 ):
     result_req = []
     for req in requirements:
@@ -112,6 +110,7 @@ def with_python_requirements(
             force_rebuild=force_rebuild,
             prefer_binary=prefer_binary,
             pre_release=pre_release,
+            no_audit_wheel=no_audit_wheel,
         )
         if isinstance(func, MarkedFunction):
             func.pythonpacks.append(pack_item)

maxframe/utils.py

@@ -19,7 +19,6 @@ import dataclasses
 import datetime
 import enum
 import functools
-import hashlib
 import importlib
 import inspect
 import io
@@ -75,7 +74,7 @@ from ._utils import (  # noqa: F401 # pylint: disable=unused-import
     tokenize_int,
 )
 from .lib.version import parse as parse_version
-from .typing_ import ChunkType, EntityType, TileableType, TimeoutType
+from .typing_ import TileableType, TimeoutType
 
 # make flake8 happy by referencing these imports
 NamedType = NamedType
@@ -245,58 +244,6 @@ def copy_tileables(tileables: List[TileableType], **kwargs):
     return op.new_tileables(inputs, kws=kws, output_limit=len(kws))
 
 
-def build_fetch_chunk(chunk: ChunkType, **kwargs) -> ChunkType:
-    from .core.operator import ShuffleProxy
-
-    chunk_op = chunk.op
-    params = chunk.params.copy()
-    assert not isinstance(chunk_op, ShuffleProxy)
-    # for non-shuffle nodes, we build Fetch chunks
-    # to replace original chunk
-    op = chunk_op.get_fetch_op_cls(chunk)(sparse=chunk.op.sparse, gpu=chunk.op.gpu)
-    return op.new_chunk(
-        None,
-        is_broadcaster=chunk.is_broadcaster,
-        kws=[params],
-        _key=chunk.key,
-        **kwargs,
-    )
-
-
-def build_fetch_tileable(tileable: TileableType) -> TileableType:
-    if tileable.is_coarse():
-        chunks = None
-    else:
-        chunks = []
-        for c in tileable.chunks:
-            fetch_chunk = build_fetch_chunk(c, index=c.index)
-            chunks.append(fetch_chunk)
-
-    tileable_op = tileable.op
-    params = tileable.params.copy()
-
-    new_op = tileable_op.get_fetch_op_cls(tileable)(_id=tileable_op.id)
-    return new_op.new_tileables(
-        None,
-        chunks=chunks,
-        nsplits=tileable.nsplits,
-        _key=tileable.key,
-        _id=tileable.id,
-        **params,
-    )[0]
-
-
-def build_fetch(entity: EntityType) -> EntityType:
-    from .core import CHUNK_TYPE, ENTITY_TYPE
-
-    if isinstance(entity, CHUNK_TYPE):
-        return build_fetch_chunk(entity)
-    elif isinstance(entity, ENTITY_TYPE):
-        return build_fetch_tileable(entity)
-    else:
-        raise TypeError(f"Type {type(entity)} not supported")
-
-
 def get_dtype(dtype: Union[np.dtype, pd.api.extensions.ExtensionDtype]):
     if pd.api.types.is_extension_array_dtype(dtype):
         return dtype
@@ -386,13 +333,7 @@ def build_temp_intermediate_table_name(session_id: str, tileable_key: str) -> st
 
 
 def build_session_volume_name(session_id: str) -> str:
-    return f"mf_vol_{session_id}"
-
-
-def build_tileable_dir_name(tileable_key: str) -> str:
-    m = hashlib.md5()
-    m.update(f"mf_dir_{tileable_key}".encode())
-    return m.hexdigest()
+    return f"mf_vol_{session_id.replace('-', '_')}"
 
 
 async def wait_http_response(
@@ -436,19 +377,27 @@ async def to_thread_pool(func, *args, pool=None, **kwargs):
     return await loop.run_in_executor(pool, func_call)
 
 
-def create_event(loop: asyncio.AbstractEventLoop) -> asyncio.Event:
+_PrimitiveType = TypeVar("_PrimitiveType")
+
+
+def create_sync_primitive(
+    cls: Type[_PrimitiveType], loop: asyncio.AbstractEventLoop
+) -> _PrimitiveType:
     """
-    Create an asyncio.Event in a certain event loop.
+    Create an asyncio sync primitive (locks, events, etc.)
+    in a certain event loop.
     """
-    if sys.version_info[1] < 10 or loop is None:
-        return asyncio.Event(loop=loop)
+    if sys.version_info[1] < 10:
+        return cls(loop=loop)
 
     # From Python3.10 the loop parameter has been removed. We should work around here.
-    old_loop = asyncio.get_running_loop()
-    asyncio.set_event_loop(loop)
-    event = asyncio.Event()
-    asyncio.set_event_loop(old_loop)
-    return event
+    old_loop = asyncio.get_event_loop()
+    try:
+        asyncio.set_event_loop(loop)
+        primitive = cls()
+    finally:
+        asyncio.set_event_loop(old_loop)
+    return primitive
 
 
 class ToThreadCancelledError(asyncio.CancelledError):
@@ -1115,3 +1064,16 @@ def get_item_if_scalar(val: Any) -> Any:
     if isinstance(val, np.ndarray) and val.shape == ():
         return val.item()
     return val
+
+
+def collect_leaf_operators(root) -> List[Type]:
+    result = []
+
+    def _collect(op_type):
+        if len(op_type.__subclasses__()) == 0:
+            result.append(op_type)
+        for subclass in op_type.__subclasses__():
+            _collect(subclass)
+
+    _collect(root)
+    return result

{maxframe-1.0.0rc1.dist-info → maxframe-1.0.0rc3.dist-info}/METADATA

@@ -1,33 +1,33 @@
 Metadata-Version: 2.1
 Name: maxframe
-Version: 1.0.0rc1
+Version: 1.0.0rc3
 Summary: MaxFrame operator-based data analyze framework
-Requires-Dist: numpy <2.0.0,>=1.19.0
-Requires-Dist: pandas >=1.0.0
-Requires-Dist: pyodps >=0.11.6.1
-Requires-Dist: scipy >=1.0
-Requires-Dist: pyarrow >=1.0.0
-Requires-Dist: msgpack >=1.0.0
-Requires-Dist: traitlets >=5.0
-Requires-Dist: cloudpickle >=1.5.0
-Requires-Dist: pyyaml >=5.1
-Requires-Dist: tornado >=6.0
-Requires-Dist: defusedxml >=0.5.0
-Requires-Dist: tqdm >=4.1.0
-Requires-Dist: importlib-metadata >=1.4
-Requires-Dist: pickle5 ; python_version < "3.8"
+Requires-Dist: numpy<2.0.0,>=1.19.0
+Requires-Dist: pandas>=1.0.0
+Requires-Dist: pyodps>=0.11.6.1
+Requires-Dist: scipy>=1.0
+Requires-Dist: pyarrow>=1.0.0
+Requires-Dist: msgpack>=1.0.0
+Requires-Dist: traitlets>=5.0
+Requires-Dist: cloudpickle<3.0.0,>=1.5.0
+Requires-Dist: pyyaml>=5.1
+Requires-Dist: tornado>=6.0
+Requires-Dist: defusedxml>=0.5.0
+Requires-Dist: tqdm>=4.1.0
+Requires-Dist: importlib-metadata>=1.4
+Requires-Dist: pickle5; python_version < "3.8"
 Provides-Extra: dev
-Requires-Dist: black >=22.3.0 ; extra == 'dev'
-Requires-Dist: flake8 >=5.0.4 ; extra == 'dev'
-Requires-Dist: pre-commit >=2.15.0 ; extra == 'dev'
-Requires-Dist: graphviz >=0.20.1 ; extra == 'dev'
+Requires-Dist: black>=22.3.0; extra == "dev"
+Requires-Dist: flake8>=5.0.4; extra == "dev"
+Requires-Dist: pre-commit>=2.15.0; extra == "dev"
+Requires-Dist: graphviz>=0.20.1; extra == "dev"
 Provides-Extra: test
-Requires-Dist: mock ; extra == 'test'
-Requires-Dist: pytest >=7.3.1 ; extra == 'test'
-Requires-Dist: pytest-cov >=4.1.0 ; extra == 'test'
-Requires-Dist: pytest-asyncio >=0.21.0 ; extra == 'test'
-Requires-Dist: pytest-timeout >=2.1.0 ; extra == 'test'
-Requires-Dist: matplotlib >=2.0.0 ; extra == 'test'
+Requires-Dist: mock; extra == "test"
+Requires-Dist: pytest>=7.3.1; extra == "test"
+Requires-Dist: pytest-cov>=4.1.0; extra == "test"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "test"
+Requires-Dist: pytest-timeout>=2.1.0; extra == "test"
+Requires-Dist: matplotlib>=2.0.0; extra == "test"
 
 MaxCompute MaxFrame Client
 ==========================