maxframe 1.0.0rc3__cp37-cp37m-win32.whl → 1.1.0__cp37-cp37m-win32.whl

maxframe/session.py

@@ -150,6 +150,10 @@ class AbstractSession(ABC):
     def session_id(self):
         return self._session_id
 
+    @property
+    def closed(self) -> bool:
+        return self._closed
+
     def __eq__(self, other):
         return (
             isinstance(other, AbstractSession)
@@ -1283,9 +1287,12 @@ def get_default_or_create(**kwargs):
         if session is None:
             # no session attached, try to create one
             warnings.warn(warning_msg)
-            session = new_session(
-                ODPS.from_global() or ODPS.from_environments(), **kwargs
+            odps_entry = (
+                kwargs.pop("odps_entry", None)
+                or ODPS.from_global()
+                or ODPS.from_environments()
             )
+            session = new_session(odps_entry=odps_entry, **kwargs)
             session.as_default()
     if isinstance(session, IsolatedAsyncSession):
         session = SyncSession.from_isolated_session(session)

maxframe/tensor/__init__.py

@@ -191,11 +191,6 @@ from .ufunc import ufunc
 # isort: off
 # noinspection PyUnresolvedReferences
 from numpy import (
-    NAN,
-    NINF,
-    AxisError,
-    Inf,
-    NaN,
     e,
     errstate,
     geterr,
@@ -206,12 +201,21 @@ from numpy import (
     seterr,
 )
 
+try:
+    from numpy.exceptions import AxisError
+except ImportError:
+    from numpy import AxisError
+
+NAN = nan
+NINF = -inf
+Inf = inf
+NaN = nan
+
 # import numpy types
 # noinspection PyUnresolvedReferences
 from numpy import (
     bool_ as bool,
     bytes_,
-    cfloat,
     character,
     complex64,
     complex128,
@@ -242,9 +246,17 @@ from numpy import (
     uint16,
     uint32,
     uint64,
-    unicode_,
     unsignedinteger,
     void,
 )
 
+try:
+    from numpy import cfloat
+except ImportError:
+    from numpy import cdouble as cfloat
+try:
+    from numpy import str_ as unicode_
+except ImportError:
+    from numpy import unicode_
+
 del fetch, ufunc

maxframe/tensor/indexing/getitem.py

@@ -130,6 +130,8 @@ def _calc_order(a, index):
             continue
         elif isinstance(ind, slice):
             shape = a.shape[in_axis]
+            if shape is np.nan:
+                return TensorOrder.C_ORDER
             slc = ind.indices(shape)
             if slc[0] == 0 and slc[1] == shape and slc[2] == 1:
                 continue

maxframe/tensor/merge/concatenate.py

@@ -26,27 +26,9 @@ class TensorConcatenate(TensorOperator, TensorOperatorMixin):
     axis = Int32Field("axis", default=0)
 
     def __call__(self, tensors):
-        if len(set(t.ndim for t in tensors)) != 1:
-            raise ValueError(
-                "all the input tensors must have same number of dimensions"
-            )
-
         axis = self.axis
-        shapes = [t.shape[:axis] + t.shape[axis + 1 :] for t in tensors]
-        if len(set(shapes)) != 1:
-            raise ValueError(
-                "all the input tensor dimensions "
-                "except for the concatenation axis must match exactly"
-            )
-
-        shape = [
-            0 if i == axis else tensors[0].shape[i] for i in range(tensors[0].ndim)
-        ]
+        shape = _calc_concatenate_shape(tensors, axis)
         shape[axis] = sum(t.shape[axis] for t in tensors)
-
-        if any(np.isnan(s) for i, s in enumerate(shape) if i != axis):
-            raise ValueError("cannot concatenate tensor with unknown shape")
-
         return self.new_tensor(tensors, shape=tuple(shape))
 
 
@@ -90,9 +72,30 @@ def concatenate(tensors, axis=0):
     if axis is None:
         axis = 0
     tensors = [astensor(t) for t in tensors]
-
     axis = validate_axis(tensors[0].ndim, axis)
+
+    if len(set(t.ndim for t in tensors)) != 1:
+        raise ValueError("all the input tensors must have same number of dimensions")
+
+    shapes = [t.shape[:axis] + t.shape[axis + 1 :] for t in tensors]
+    if len(set(shapes)) != 1:
+        raise ValueError(
+            "all the input tensor dimensions "
+            "except for the concatenation axis must match exactly"
+        )
+    shape = _calc_concatenate_shape(tensors, axis)
+    if any(np.isnan(s) for i, s in enumerate(shape) if i != axis):
+        raise ValueError("cannot concatenate tensor with unknown shape")
+
+    return _concatenate(tensors, axis)
+
+
+def _concatenate(tensors, axis=0):
     dtype = np.result_type(*(t.dtype for t in tensors))
 
     op = TensorConcatenate(axis=axis, dtype=dtype)
     return op(tensors)
+
+
+def _calc_concatenate_shape(tensors, axis):
+    return [0 if i == axis else tensors[0].shape[i] for i in range(tensors[0].ndim)]

maxframe/tensor/merge/vstack.py

@@ -14,7 +14,7 @@
 
 
 from ..misc import atleast_2d
-from .concatenate import concatenate
+from .concatenate import _concatenate, concatenate
 
 
 def vstack(tup):
@@ -68,3 +68,7 @@ def vstack(tup):
 
     """
     return concatenate([atleast_2d(t) for t in tup], axis=0)
+
+
+def _vstack(tup):
+    return _concatenate([atleast_2d(t) for t in tup], axis=0)

maxframe/tensor/misc/transpose.py

@@ -125,5 +125,5 @@ def transpose(a, axes=None):
         axes = list(range(a.ndim))[::-1]
     else:
         axes = list(axes)
-    op = TensorTranspose(axes)
+    op = TensorTranspose(axes, dtype=a.dtype)
     return op(a)

maxframe/tests/utils.py

@@ -18,11 +18,13 @@ import hashlib
 import os
 import queue
 import socket
+import time
 import types
 from threading import Thread
 from typing import Dict, List, Optional, Set, Tuple
 
 import pytest
+from odps import ODPS
 from tornado import netutil
 
 from ..core import Tileable, TileableGraph
@@ -171,3 +173,17 @@ def get_test_unique_name(size=None):
     if size:
         digest = digest[:size]
     return digest + "_" + str(os.getpid())
+
+
+def assert_mf_index_dtype(idx_obj, dtype):
+    from ..dataframe.core import IndexValue
+
+    assert isinstance(idx_obj, IndexValue.IndexBase) and idx_obj.dtype == dtype
+
+
+def ensure_table_deleted(odps_entry: ODPS, table_name: str) -> None:
+    retry_times = 20
+    while odps_entry.exist_table(table_name) and retry_times > 0:
+        time.sleep(1)
+        retry_times -= 1
+    assert not odps_entry.exist_table(table_name)

maxframe/udf.py

@@ -19,6 +19,7 @@ from odps.models import Resource
 
 from .serialization.serializables import (
     BoolField,
+    DictField,
     FieldTypes,
     FunctionField,
     ListField,
@@ -54,6 +55,10 @@ class MarkedFunction(Serializable):
     func = FunctionField("func")
     resources = ListField("resources", FieldTypes.string, default_factory=list)
     pythonpacks = ListField("pythonpacks", FieldTypes.reference, default_factory=list)
+    expect_engine = StringField("expect_engine", default=None)
+    expect_resources = DictField(
+        "expect_resources", FieldTypes.string, default_factory=dict
+    )
 
     def __init__(self, func: Optional[Callable] = None, **kw):
         super().__init__(func=func, **kw)
@@ -120,6 +125,28 @@ def with_python_requirements(
     return func_wrapper
 
 
+def with_running_options(
+    *,
+    engine: Optional[str] = None,
+    cpu: Optional[int] = None,
+    memory: Optional[int] = None,
+    **kwargs,
+):
+    engine = engine.upper() if engine else None
+    resources = {"cpu": cpu, "memory": memory, **kwargs}
+
+    def func_wrapper(func):
+        if all(v is None for v in (engine, cpu, memory)):
+            return func
+        if isinstance(func, MarkedFunction):
+            func.expect_engine = engine
+            func.expect_resources = resources
+            return func
+        return MarkedFunction(func, expect_engine=engine, expect_resources=resources)
+
+    return func_wrapper
+
+
 with_resource_libraries = with_resources
 
 

maxframe/utils.py

@@ -370,13 +370,6 @@ def format_timeout_params(timeout: TimeoutType) -> str:
         return f"?wait=1&timeout={timeout}"
 
 
-async def to_thread_pool(func, *args, pool=None, **kwargs):
-    loop = asyncio.events.get_running_loop()
-    ctx = contextvars.copy_context()
-    func_call = functools.partial(ctx.run, func, *args, **kwargs)
-    return await loop.run_in_executor(pool, func_call)
-
-
 _PrimitiveType = TypeVar("_PrimitiveType")
 
 
@@ -438,15 +431,22 @@ class ToThreadMixin:
                 thread_name_prefix=f"{type(self).__name__}Pool-{self._counter()}",
             )
 
-        task = asyncio.create_task(
-            to_thread_pool(func, *args, **kwargs, pool=self._pool)
-        )
+        loop = asyncio.events.get_running_loop()
+        ctx = contextvars.copy_context()
+        func_call = functools.partial(ctx.run, func, *args, **kwargs)
+        fut = loop.run_in_executor(self._pool, func_call)
+
         try:
-            return await asyncio.wait_for(asyncio.shield(task), timeout)
+            coro = fut
+            if wait_on_cancel:
+                coro = asyncio.shield(coro)
+            if timeout is not None:
+                coro = asyncio.wait_for(coro, timeout)
+            return await coro
         except (asyncio.CancelledError, asyncio.TimeoutError) as ex:
             if not wait_on_cancel:
                 raise
-            result = await task
+            result = await fut
             raise ToThreadCancelledError(*ex.args, result=result)
 
     def ensure_async_call(
@@ -835,8 +835,41 @@ def parse_readable_size(value: Union[str, int, float]) -> Tuple[float, bool]:
         raise ValueError(f"Unknown limitation value: {value}")
 
 
-def remove_suffix(value: str, suffix: str) -> str:
-    return value[: -len(suffix)] if value.endswith(suffix) else value
+def remove_suffix(value: str, suffix: str) -> Tuple[str, bool]:
+    """
+    Remove a suffix from a given string if it exists.
+
+    Parameters
+    ----------
+    value : str
+        The original string.
+    suffix : str
+        The suffix to be removed.
+
+    Returns
+    -------
+    Tuple[str, bool]
+        A tuple containing the modified string and a boolean indicating whether the suffix was found.
+    """
+
+    # Check if the suffix is an empty string
+    if len(suffix) == 0:
+        # If the suffix is empty, return the original string with True
+        return value, True
+
+    # Check if the length of the value is less than the length of the suffix
+    if len(value) < len(suffix):
+        # If the value is shorter than the suffix, it cannot have the suffix
+        return value, False
+
+    # Check if the suffix matches the end of the value
+    match = value.endswith(suffix)
+
+    # If the suffix is found, remove it; otherwise, return the original string
+    if match:
+        return value[: -len(suffix)], match
+    else:
+        return value, match
 
 
 def find_objects(nested: Union[List, Dict], types: Union[Type, Tuple[Type]]) -> List:
@@ -1077,3 +1110,20 @@ def collect_leaf_operators(root) -> List[Type]:
 
     _collect(root)
     return result
+
+
+@contextmanager
+def sync_pyodps_options():
+    from odps.config import option_context as pyodps_option_context
+
+    from .config import options
+
+    with pyodps_option_context() as cfg:
+        cfg.local_timezone = options.local_timezone
+        if options.session.enable_schema:
+            cfg.enable_schema = options.session.enable_schema
+        yield
+
+
+def str_to_bool(s: Optional[str]) -> Optional[bool]:
+    return s.lower().strip() in ("true", "1") if s is not None else None

{maxframe-1.0.0rc3.dist-info → maxframe-1.1.0.dist-info}/METADATA

@@ -1,10 +1,10 @@
 Metadata-Version: 2.1
 Name: maxframe
-Version: 1.0.0rc3
+Version: 1.1.0
 Summary: MaxFrame operator-based data analyze framework
 Requires-Dist: numpy <2.0.0,>=1.19.0
 Requires-Dist: pandas >=1.0.0
-Requires-Dist: pyodps >=0.11.6.1
+Requires-Dist: pyodps >=0.12.0
 Requires-Dist: scipy >=1.0
 Requires-Dist: pyarrow >=1.0.0
 Requires-Dist: msgpack >=1.0.0