maxframe 1.0.0rc2__cp311-cp311-win32.whl → 1.0.0rc4__cp311-cp311-win32.whl

maxframe/tensor/{base → misc}/transpose.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 1999-2024 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -66,33 +64,39 @@ class TensorTranspose(TensorHasInput, TensorOperatorMixin):
 
 def transpose(a, axes=None):
     """
-    Permute the dimensions of a tensor.
+    Returns an array with axes transposed.
+
+    For a 1-D array, this returns an unchanged view of the original array, as a
+    transposed vector is simply the same vector.
+    To convert a 1-D array into a 2-D column vector, an additional dimension
+    must be added, e.g., ``mt.atleast_2d(a).T`` achieves this, as does
+    ``a[:, mt.newaxis]``.
+    For a 2-D array, this is the standard matrix transpose.
+    For an n-D array, if axes are given, their order indicates how the
+    axes are permuted (see Examples). If axes are not provided, then
+    ``transpose(a).shape == a.shape[::-1]``.
 
     Parameters
     ----------
     a : array_like
-        Input tensor.
-    axes : list of ints, optional
-        By default, reverse the dimensions, otherwise permute the axes
-        according to the values given.
+        Input array.
+    axes : tuple or list of ints, optional
+        If specified, it must be a tuple or list which contains a permutation
+        of [0,1,...,N-1] where N is the number of axes of `a`. The `i`'th axis
+        of the returned array will correspond to the axis numbered ``axes[i]``
+        of the input. If not specified, defaults to ``range(a.ndim)[::-1]``,
+        which reverses the order of the axes.
 
     Returns
     -------
-    p : Tensor
-        `a` with its axes permuted.  A view is returned whenever
-        possible.
-
-    See Also
-    --------
-    moveaxis
-    argsort
+    p : ndarray
+        `a` with its axes permuted. A view is returned whenever possible.
 
     Notes
     -----
-    Use `transpose(a, argsort(axes))` to invert the transposition of tensors
+    Use ``transpose(a, argsort(axes))`` to invert the transposition of tensors
     when using the `axes` keyword argument.
 
-    Transposing a 1-D array returns an unchanged view of the original tensor.
 
     Examples
     --------
@@ -121,5 +125,5 @@ def transpose(a, axes=None):
         axes = list(range(a.ndim))[::-1]
     else:
         axes = list(axes)
-    op = TensorTranspose(axes, dtype=a.dtype, sparse=a.issparse())
+    op = TensorTranspose(axes, dtype=a.dtype)
     return op(a)

maxframe/tensor/operators.py

@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 from ..core import OutputType
 from ..core.operator import (
-    Fuse,
     HasInput,
     MapReduceOperator,
     Operator,
@@ -115,9 +115,3 @@ class TensorMapReduceOperator(MapReduceOperator):
     _output_type_ = OutputType.tensor
 
     dtype = DataTypeField("dtype", default=None)
-
-
-class TensorFuse(Fuse):
-    _output_type_ = OutputType.tensor
-
-    dtype = DataTypeField("dtype", default=None)

maxframe/tensor/random/core.py

@@ -19,9 +19,9 @@ from contextlib import contextmanager
 import numpy as np
 
 from ...serialization.serializables import FieldTypes, Int32Field, TupleField
-from ..base import broadcast_to
 from ..core import TENSOR_TYPE
 from ..datasource import tensor as astensor
+from ..misc import broadcast_to
 from ..operators import TensorMapReduceOperator, TensorOperator, TensorOperatorMixin
 from ..utils import broadcast_shape
 

maxframe/tensor/reduction/count_nonzero.py

@@ -22,6 +22,7 @@ from .core import TensorReduction, TensorReductionMixin
 
 class TensorCountNonzero(TensorReduction, TensorReductionMixin):
     _op_type_ = opcodes.COUNT_NONZERO
+    _func_name = "count_nonzero"
 
     def __init__(self, dtype=None, **kw):
         if dtype is None:

maxframe/tensor/reduction/mean.py

@@ -23,6 +23,7 @@ from .core import TensorReduction, TensorReductionMixin
 
 class TensorMean(TensorReduction, TensorReductionMixin):
     _op_type_ = opcodes.MEAN
+    _func_name = "mean"
 
 
 def mean(a, axis=None, dtype=None, out=None, keepdims=None):

maxframe/tensor/reduction/nanmean.py

@@ -23,6 +23,7 @@ from .core import TensorReduction, TensorReductionMixin
 
 class TensorNanMean(TensorReduction, TensorReductionMixin):
     _op_type_ = opcodes.NANMEAN
+    _func_name = "nanmean"
 
 
 def nanmean(a, axis=None, dtype=None, out=None, keepdims=None):

maxframe/tensor/reduction/nanvar.py

@@ -24,6 +24,7 @@ from .core import TensorReduction, TensorReductionMixin
 
 class TensorNanMoment(TensorReduction, TensorReductionMixin):
     _op_type_ = opcodes.NANMOMENT
+    _func_name = "nanvar"
 
     moment = Int32Field("moment", default=2)
     ddof = Int32Field("ddof", default=None)
@@ -36,6 +37,7 @@ class TensorNanMoment(TensorReduction, TensorReductionMixin):
 
 class TensorNanVar(TensorReduction, TensorReductionMixin):
     _op_type_ = opcodes.NANVAR
+    _func_name = "nanvar"
 
     ddof = Int32Field("ddof", default=0)
 

maxframe/tensor/reduction/tests/test_reduction.py

@@ -17,8 +17,11 @@
 import numpy as np
 import pytest
 
+from maxframe.tensor.reduction.core import TensorReduction
+
+from ....utils import collect_leaf_operators
 from ...datasource import ones, tensor
-from .. import all
+from .. import *  # noqa: F401
 
 
 def test_base_reduction():
@@ -179,3 +182,11 @@ def test_var_reduction():
 
     res1 = var(ones((10, 8, 8), chunk_size=3), axis=1)
     assert res1.shape == (10, 8)
+
+
+def test_reduction_op_func_name():
+    # make sure all the binary op has defined the func name.
+
+    results = collect_leaf_operators(TensorReduction)
+    for op_type in results:
+        assert hasattr(op_type, "_func_name")

maxframe/tensor/reduction/var.py

@@ -42,6 +42,7 @@ def reduce_var_square(var_square, avg_diff, count, op, axis, sum_func):
 
 class TensorMoment(TensorReduction, TensorReductionMixin):
     _op_type_ = opcodes.MOMENT
+    _func_name = "var"
 
     moment = Int32Field("moment", default=2)
     ddof = Int32Field("ddof", default=None)
@@ -54,6 +55,7 @@ class TensorMoment(TensorReduction, TensorReductionMixin):
 
 class TensorVar(TensorReduction, TensorReductionMixin):
     _op_type_ = opcodes.VAR
+    _func_name = "var"
 
     ddof = Int32Field("ddof", default=0)
 

maxframe/tensor/utils.py

@@ -19,18 +19,13 @@ import itertools
 import operator
 from collections import OrderedDict
 from collections.abc import Iterable
-from functools import lru_cache, wraps
+from functools import wraps
 from math import ceil
 from numbers import Integral
 from typing import Dict, List, Union
 
 import numpy as np
 
-try:
-    import tiledb
-except (ImportError, OSError):  # pragma: no cover
-    tildb = None
-
 from ..core import ExecutableTuple
 from ..lib.mmh3 import hash_from_buffer
 from ..utils import lazy_import
@@ -508,7 +503,7 @@ def decide_unify_split(*splits):
 
 
 def check_out_param(out, t, casting):
-    from .base import broadcast_to
+    from .misc import broadcast_to
 
     if not hasattr(out, "shape"):
         raise TypeError("return arrays must be a tensor")
@@ -563,21 +558,6 @@ def filter_inputs(inputs):
     return [inp for inp in inputs if isinstance(inp, ENTITY_TYPE)]
 
 
-# As TileDB Ctx's creation is a bit time-consuming,
-# we just cache the Ctx
-# also remember the arguments should be hashable
-@lru_cache(10)
-def _create_tiledb_ctx(conf_tuple):
-    if conf_tuple is not None:
-        return tiledb.Ctx(dict(conf_tuple))
-    return tiledb.Ctx()
-
-
-def get_tiledb_ctx(conf):
-    key = tuple(conf.items()) if conf is not None else None
-    return _create_tiledb_ctx(key)
-
-
 # this function is only used for pandas' compatibility
 def to_numpy(pdf):
     try:

maxframe/typing_.py

@@ -12,11 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import TypeVar, Union
+from numbers import Integral
+from typing import List, TypeVar, Union
 
 import pandas as pd
 import pyarrow as pa
 
+SlicesType = List[Union[None, Integral, slice]]
+
 TimeoutType = Union[int, float, None]
 
 

maxframe/udf.py

@@ -29,28 +29,25 @@ from .utils import tokenize
 
 
 class PythonPackOptions(Serializable):
+    _key_args = ("force_rebuild", "prefer_binary", "pre_release", "no_audit_wheel")
+
     key = StringField("key")
     requirements = ListField("requirements", FieldTypes.string, default_factory=list)
     force_rebuild = BoolField("force_rebuild", default=False)
     prefer_binary = BoolField("prefer_binary", default=False)
     pre_release = BoolField("pre_release", default=False)
     pack_instance_id = StringField("pack_instance_id", default=None)
+    no_audit_wheel = BoolField("no_audit_wheel", default=False)
 
     def __init__(self, key: str = None, **kw):
         super().__init__(key=key, **kw)
         if self.key is None:
-            args = {
-                "force_rebuild": self.force_rebuild,
-                "prefer_binary": self.prefer_binary,
-                "pre_release": self.pre_release,
-            }
+            args = {k: getattr(self, k) for k in self._key_args}
             self.key = tokenize(set(self.requirements), args)
 
     def __repr__(self):
-        return (
-            f"<PythonPackOptions {self.requirements} force_rebuild={self.force_rebuild} "
-            f"prefer_binary={self.prefer_binary} pre_release={self.pre_release}>"
-        )
+        args_str = " ".join(f"{k}={getattr(self, k)}" for k in self._key_args)
+        return f"<PythonPackOptions {self.requirements} {args_str}>"
 
 
 class MarkedFunction(Serializable):
@@ -101,6 +98,7 @@ def with_python_requirements(
     force_rebuild: bool = False,
     prefer_binary: bool = False,
     pre_release: bool = False,
+    no_audit_wheel: bool = False,
 ):
     result_req = []
     for req in requirements:
@@ -112,6 +110,7 @@ def with_python_requirements(
             force_rebuild=force_rebuild,
             prefer_binary=prefer_binary,
             pre_release=pre_release,
+            no_audit_wheel=no_audit_wheel,
         )
         if isinstance(func, MarkedFunction):
             func.pythonpacks.append(pack_item)

maxframe/utils.py

@@ -19,7 +19,6 @@ import dataclasses
 import datetime
 import enum
 import functools
-import hashlib
 import importlib
 import inspect
 import io
@@ -75,7 +74,7 @@ from ._utils import (  # noqa: F401 # pylint: disable=unused-import
     tokenize_int,
 )
 from .lib.version import parse as parse_version
-from .typing_ import ChunkType, EntityType, TileableType, TimeoutType
+from .typing_ import TileableType, TimeoutType
 
 # make flake8 happy by referencing these imports
 NamedType = NamedType
@@ -245,58 +244,6 @@ def copy_tileables(tileables: List[TileableType], **kwargs):
     return op.new_tileables(inputs, kws=kws, output_limit=len(kws))
 
 
-def build_fetch_chunk(chunk: ChunkType, **kwargs) -> ChunkType:
-    from .core.operator import ShuffleProxy
-
-    chunk_op = chunk.op
-    params = chunk.params.copy()
-    assert not isinstance(chunk_op, ShuffleProxy)
-    # for non-shuffle nodes, we build Fetch chunks
-    # to replace original chunk
-    op = chunk_op.get_fetch_op_cls(chunk)(sparse=chunk.op.sparse, gpu=chunk.op.gpu)
-    return op.new_chunk(
-        None,
-        is_broadcaster=chunk.is_broadcaster,
-        kws=[params],
-        _key=chunk.key,
-        **kwargs,
-    )
-
-
-def build_fetch_tileable(tileable: TileableType) -> TileableType:
-    if tileable.is_coarse():
-        chunks = None
-    else:
-        chunks = []
-        for c in tileable.chunks:
-            fetch_chunk = build_fetch_chunk(c, index=c.index)
-            chunks.append(fetch_chunk)
-
-    tileable_op = tileable.op
-    params = tileable.params.copy()
-
-    new_op = tileable_op.get_fetch_op_cls(tileable)(_id=tileable_op.id)
-    return new_op.new_tileables(
-        None,
-        chunks=chunks,
-        nsplits=tileable.nsplits,
-        _key=tileable.key,
-        _id=tileable.id,
-        **params,
-    )[0]
-
-
-def build_fetch(entity: EntityType) -> EntityType:
-    from .core import CHUNK_TYPE, ENTITY_TYPE
-
-    if isinstance(entity, CHUNK_TYPE):
-        return build_fetch_chunk(entity)
-    elif isinstance(entity, ENTITY_TYPE):
-        return build_fetch_tileable(entity)
-    else:
-        raise TypeError(f"Type {type(entity)} not supported")
-
-
 def get_dtype(dtype: Union[np.dtype, pd.api.extensions.ExtensionDtype]):
     if pd.api.types.is_extension_array_dtype(dtype):
         return dtype
@@ -386,13 +333,7 @@ def build_temp_intermediate_table_name(session_id: str, tileable_key: str) -> st
 
 
 def build_session_volume_name(session_id: str) -> str:
-    return f"mf_vol_{session_id}"
-
-
-def build_tileable_dir_name(tileable_key: str) -> str:
-    m = hashlib.md5()
-    m.update(f"mf_dir_{tileable_key}".encode())
-    return m.hexdigest()
+    return f"mf_vol_{session_id.replace('-', '_')}"
 
 
 async def wait_http_response(
@@ -429,13 +370,6 @@ def format_timeout_params(timeout: TimeoutType) -> str:
         return f"?wait=1&timeout={timeout}"
 
 
-async def to_thread_pool(func, *args, pool=None, **kwargs):
-    loop = asyncio.events.get_running_loop()
-    ctx = contextvars.copy_context()
-    func_call = functools.partial(ctx.run, func, *args, **kwargs)
-    return await loop.run_in_executor(pool, func_call)
-
-
 _PrimitiveType = TypeVar("_PrimitiveType")
 
 
@@ -497,15 +431,22 @@ class ToThreadMixin:
                 thread_name_prefix=f"{type(self).__name__}Pool-{self._counter()}",
             )
 
-        task = asyncio.create_task(
-            to_thread_pool(func, *args, **kwargs, pool=self._pool)
-        )
+        loop = asyncio.events.get_running_loop()
+        ctx = contextvars.copy_context()
+        func_call = functools.partial(ctx.run, func, *args, **kwargs)
+        fut = loop.run_in_executor(self._pool, func_call)
+
         try:
-            return await asyncio.wait_for(asyncio.shield(task), timeout)
+            coro = fut
+            if wait_on_cancel:
+                coro = asyncio.shield(coro)
+            if timeout is not None:
+                coro = asyncio.wait_for(coro, timeout)
+            return await coro
         except (asyncio.CancelledError, asyncio.TimeoutError) as ex:
             if not wait_on_cancel:
                 raise
-            result = await task
+            result = await fut
             raise ToThreadCancelledError(*ex.args, result=result)
 
     def ensure_async_call(
@@ -1123,3 +1064,38 @@ def get_item_if_scalar(val: Any) -> Any:
     if isinstance(val, np.ndarray) and val.shape == ():
         return val.item()
     return val
+
+
+def collect_leaf_operators(root) -> List[Type]:
+    result = []
+
+    def _collect(op_type):
+        if len(op_type.__subclasses__()) == 0:
+            result.append(op_type)
+        for subclass in op_type.__subclasses__():
+            _collect(subclass)
+
+    _collect(root)
+    return result
+
+
+@contextmanager
+def sync_pyodps_options():
+    from odps.config import OptionError
+    from odps.config import option_context as pyodps_option_context
+
+    from .config import options
+
+    with pyodps_option_context() as cfg:
+        cfg.local_timezone = options.local_timezone
+        if options.session.enable_schema:
+            try:
+                cfg.enable_schema = options.session.enable_schema
+            except (AttributeError, OptionError):
+                # fixme enable_schema only supported in PyODPS 0.12.0 or later
+                cfg.always_enable_schema = options.session.enable_schema
+        yield
+
+
+def str_to_bool(s: Optional[str]) -> Optional[bool]:
+    return s.lower().strip() in ("true", "1") if s is not None else None

maxframe-1.0.0rc4.dist-info/METADATA

@@ -0,0 +1,104 @@
+Metadata-Version: 2.1
+Name: maxframe
+Version: 1.0.0rc4
+Summary: MaxFrame operator-based data analyze framework
+Requires-Dist: numpy<2.0.0,>=1.19.0
+Requires-Dist: pandas>=1.0.0
+Requires-Dist: pyodps>=0.11.6.1
+Requires-Dist: scipy>=1.0
+Requires-Dist: pyarrow>=1.0.0
+Requires-Dist: msgpack>=1.0.0
+Requires-Dist: traitlets>=5.0
+Requires-Dist: cloudpickle<3.0.0,>=1.5.0
+Requires-Dist: pyyaml>=5.1
+Requires-Dist: tornado>=6.0
+Requires-Dist: defusedxml>=0.5.0
+Requires-Dist: tqdm>=4.1.0
+Requires-Dist: importlib-metadata>=1.4
+Requires-Dist: pickle5; python_version < "3.8"
+Provides-Extra: dev
+Requires-Dist: black>=22.3.0; extra == "dev"
+Requires-Dist: flake8>=5.0.4; extra == "dev"
+Requires-Dist: pre-commit>=2.15.0; extra == "dev"
+Requires-Dist: graphviz>=0.20.1; extra == "dev"
+Provides-Extra: test
+Requires-Dist: mock; extra == "test"
+Requires-Dist: pytest>=7.3.1; extra == "test"
+Requires-Dist: pytest-cov>=4.1.0; extra == "test"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "test"
+Requires-Dist: pytest-timeout>=2.1.0; extra == "test"
+Requires-Dist: matplotlib>=2.0.0; extra == "test"
+
+MaxCompute MaxFrame Client
+==========================
+
+MaxFrame is a computational framework created by Alibaba Cloud to
+provide a way for Python developers to parallelize their code with
+MaxCompute. It creates a runnable computation graph locally, submits it
+to MaxCompute to execute and obtains results from MaxCompute.
+
+MaxFrame client is the client of MaxFrame. Currently it provides a
+DataFrame-based SDK with compatible APIs for pandas. In future, other
+common Python libraries like numpy and scikit-learn will be added as
+well. Python 3.7 is recommended for MaxFrame client to enable all
+functionalities while supports for higher Python versions are on the
+way.
+
+Installation
+------------
+
+You may install MaxFrame client through PIP:
+
+.. code:: bash
+
+   pip install maxframe
+
+Latest beta version can be installed with ``--pre`` argument:
+
+.. code:: bash
+
+   pip install --pre maxframe
+
+You can also install MaxFrame client from source code:
+
+.. code:: bash
+
+   pip install git+https://github.com/aliyun/alibabacloud-odps-maxframe-client.git
+
+Getting started
+---------------
+
+We show a simple code example of MaxFrame client which read data from a
+MaxCompute table, performs some simple data transform and writes back
+into MaxCompute.
+
+.. code:: python
+
+   import maxframe.dataframe as md
+   import os
+   from maxframe import new_session
+   from odps import ODPS
+
+   o = ODPS(
+       os.getenv('ALIBABA_CLOUD_ACCESS_KEY_ID'),
+       os.getenv('ALIBABA_CLOUD_ACCESS_KEY_SECRET'),
+       project='your-default-project',
+       endpoint='your-end-point',
+   )
+   session = new_session(o)
+
+   df = md.read_odps_table("source_table")
+   df["A"] = "prefix_" + df["A"]
+   md.to_odps_table(df, "prefix_source_table")
+
+Documentation
+-------------
+
+Detailed documentations can be found
+`here <https://maxframe.readthedocs.io>`__.
+
+License
+-------
+
+Licensed under the `Apache License
+2.0 <https://www.apache.org/licenses/LICENSE-2.0.html>`__.