maxframe 1.0.0rc1__cp37-cp37m-win32.whl → 1.0.0rc2__cp37-cp37m-win32.whl

maxframe/odpsio/tests/test_tableio.py

@@ -12,22 +12,37 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import datetime
+
 import numpy as np
 import pandas as pd
 import pyarrow as pa
+import pytest
 from odps import ODPS
 
+from ...config import options
 from ...tests.utils import flaky, tn
 from ...utils import config_odps_default_options
-from ..tableio import HaloTableIO
+from ..tableio import ODPSTableIO
+
+
+@pytest.fixture
+def switch_table_io(request):
+    old_use_common_table = options.use_common_table
+    try:
+        options.use_common_table = request.param
+        yield
+    finally:
+        options.use_common_table = old_use_common_table
 
 
 @flaky(max_runs=3)
-def test_empty_table_io():
+@pytest.mark.parametrize("switch_table_io", [False, True], indirect=True)
+def test_empty_table_io(switch_table_io):
     config_odps_default_options()
 
     o = ODPS.from_environments()
-    halo_table_io = HaloTableIO(o)
+    table_io = ODPSTableIO(o)
 
     # test read from empty table
     empty_table_name = tn("test_empty_table_halo_read")
@@ -35,42 +50,53 @@ def test_empty_table_io():
     tb = o.create_table(empty_table_name, "col1 string", lifecycle=1)
 
     try:
-        with halo_table_io.open_reader(empty_table_name) as reader:
+        with table_io.open_reader(empty_table_name) as reader:
             assert len(reader.read_all()) == 0
     finally:
         tb.drop()
 
 
 @flaky(max_runs=3)
-def test_table_io_without_parts():
+@pytest.mark.parametrize("switch_table_io", [False, True], indirect=True)
+def test_table_io_without_parts(switch_table_io):
     config_odps_default_options()
 
     o = ODPS.from_environments()
-    halo_table_io = HaloTableIO(o)
+    table_io = ODPSTableIO(o)
 
     # test read and write tables without partition
     no_part_table_name = tn("test_no_part_halo_write")
     o.delete_table(no_part_table_name, if_exists=True)
-    tb = o.create_table(
-        no_part_table_name, ",".join(f"{c} double" for c in "abcde"), lifecycle=1
-    )
+    col_desc = ",".join(f"{c} double" for c in "abcde") + ", f datetime"
+    tb = o.create_table(no_part_table_name, col_desc, lifecycle=1)
 
     try:
         pd_data = pd.DataFrame(np.random.rand(100, 5), columns=list("abcde"))
-        with halo_table_io.open_writer(no_part_table_name) as writer:
+        date_val = [
+            (
+                datetime.datetime.now().replace(microsecond=0)
+                + datetime.timedelta(seconds=i)
+            )
+            for i in range(100)
+        ]
+        pd_data["f"] = pd.Series(date_val, dtype="datetime64[ms]").dt.tz_localize(
+            options.local_timezone
+        )
+        with table_io.open_writer(no_part_table_name) as writer:
             writer.write(pa.Table.from_pandas(pd_data, preserve_index=False))
-        with halo_table_io.open_reader(no_part_table_name) as reader:
+        with table_io.open_reader(no_part_table_name) as reader:
             pd.testing.assert_frame_equal(reader.read_all().to_pandas(), pd_data)
     finally:
         tb.drop()
 
 
 @flaky(max_runs=3)
-def test_table_io_with_range_reader():
+@pytest.mark.parametrize("switch_table_io", [False, True], indirect=True)
+def test_table_io_with_range_reader(switch_table_io):
     config_odps_default_options()
 
     o = ODPS.from_environments()
-    halo_table_io = HaloTableIO(o)
+    table_io = ODPSTableIO(o)
 
     # test read and write tables without partition
     no_part_table_name = tn("test_no_part_halo_write")
@@ -81,15 +107,15 @@ def test_table_io_with_range_reader():
 
     try:
         pd_data = pd.DataFrame(np.random.rand(100, 5), columns=list("abcde"))
-        with halo_table_io.open_writer(no_part_table_name) as writer:
+        with table_io.open_writer(no_part_table_name) as writer:
             writer.write(pa.Table.from_pandas(pd_data, preserve_index=False))
 
-        with halo_table_io.open_reader(
+        with table_io.open_reader(
             no_part_table_name, start=None, stop=100, row_batch_size=10
         ) as reader:
             pd.testing.assert_frame_equal(reader.read_all().to_pandas(), pd_data)
 
-        with halo_table_io.open_reader(
+        with table_io.open_reader(
             no_part_table_name,
             start=-2,
             stop=-52,
@@ -105,11 +131,12 @@ def test_table_io_with_range_reader():
 
 
 @flaky(max_runs=3)
-def test_table_io_with_parts():
+@pytest.mark.parametrize("switch_table_io", [False, True], indirect=True)
+def test_table_io_with_parts(switch_table_io):
     config_odps_default_options()
 
     o = ODPS.from_environments()
-    halo_table_io = HaloTableIO(o)
+    table_io = ODPSTableIO(o)
 
     # test read and write tables with partition
     parted_table_name = tn("test_parted_halo_write")
@@ -122,11 +149,11 @@ def test_table_io_with_parts():
 
     try:
         pd_data = pd.DataFrame(np.random.rand(100, 5), columns=list("abcde"))
-        with halo_table_io.open_writer(parted_table_name, "pt=test") as writer:
+        with table_io.open_writer(parted_table_name, "pt=test") as writer:
             writer.write(pa.Table.from_pandas(pd_data, preserve_index=False))
-        with halo_table_io.open_reader(parted_table_name, "pt=test") as reader:
+        with table_io.open_reader(parted_table_name, "pt=test") as reader:
             pd.testing.assert_frame_equal(reader.read_all().to_pandas(), pd_data)
-        with halo_table_io.open_reader(
+        with table_io.open_reader(
             parted_table_name, "pt=test", partition_columns=True
         ) as reader:
             expected_data = pd_data.copy()

maxframe/protocol.py

@@ -154,6 +154,9 @@ class ODPSTableResultInfo(ResultInfo):
     partition_specs: Optional[List[str]] = ListField(
         "partition_specs", FieldTypes.string, default=None
     )
+    table_meta: Optional["DataFrameTableMeta"] = ReferenceField(
+        "table_meta", default=None
+    )
 
     def __init__(self, result_type: ResultType = None, **kw):
         result_type = result_type or ResultType.ODPS_TABLE
@@ -164,8 +167,17 @@ class ODPSTableResultInfo(ResultInfo):
         ret["full_table_name"] = self.full_table_name
         if self.partition_specs:
             ret["partition_specs"] = self.partition_specs
+        if self.table_meta:
+            ret["table_meta"] = self.table_meta.to_json()
         return ret
 
+    @classmethod
+    def _json_to_kwargs(cls, serialized: dict) -> dict:
+        kw = super()._json_to_kwargs(serialized)
+        if "table_meta" in kw:
+            kw["table_meta"] = DataFrameTableMeta.from_json(kw["table_meta"])
+        return kw
+
 
 class ODPSVolumeResultInfo(ResultInfo):
     _result_type = ResultType.ODPS_VOLUME
@@ -469,7 +481,7 @@ class DecrefRequest(Serializable):
     keys: List[str] = ListField("keys", FieldTypes.string, default=None)
 
 
-class DataFrameTableMeta(Serializable):
+class DataFrameTableMeta(JsonSerializable):
     __slots__ = "_pd_column_names", "_pd_index_level_names"
 
     table_name: Optional[str] = StringField("table_name", default=None)
@@ -500,7 +512,7 @@ class DataFrameTableMeta(Serializable):
             self._pd_index_level_names = self.pd_index_dtypes.index.tolist()
             return self._pd_index_level_names
 
-    def __eq__(self, other: "Serializable") -> bool:
+    def __eq__(self, other: "DataFrameTableMeta") -> bool:
         if not isinstance(other, type(self)):
             return False
         for k in self._FIELDS:
@@ -511,3 +523,29 @@ class DataFrameTableMeta(Serializable):
             if not is_same:
                 return False
         return True
+
+    def to_json(self) -> dict:
+        b64_pk = lambda x: base64.b64encode(pickle.dumps(x))
+        ret = {
+            "table_name": self.table_name,
+            "type": self.type.value,
+            "table_column_names": self.table_column_names,
+            "table_index_column_names": self.table_index_column_names,
+            "pd_column_dtypes": b64_pk(self.pd_column_dtypes),
+            "pd_column_level_names": b64_pk(self.pd_column_level_names),
+            "pd_index_dtypes": b64_pk(self.pd_index_dtypes),
+        }
+        return ret
+
+    @classmethod
+    def from_json(cls, serialized: dict) -> "DataFrameTableMeta":
+        b64_upk = lambda x: pickle.loads(base64.b64decode(x))
+        serialized.update(
+            {
+                "type": OutputType(serialized["type"]),
+                "pd_column_dtypes": b64_upk(serialized["pd_column_dtypes"]),
+                "pd_column_level_names": b64_upk(serialized["pd_column_level_names"]),
+                "pd_index_dtypes": b64_upk(serialized["pd_index_dtypes"]),
+            }
+        )
+        return DataFrameTableMeta(**serialized)

maxframe/serialization/serializables/core.py

@@ -51,7 +51,10 @@ def _is_field_primitive_compound(field: Field):
 class SerializableMeta(type):
     def __new__(mcs, name: str, bases: Tuple[Type], properties: Dict):
         # All the fields including misc fields.
-        name_hash = hash(f"{properties.get('__module__')}.{name}")
+        legacy_name_hash = hash(f"{properties.get('__module__')}.{name}")
+        name_hash = hash(
+            f"{properties.get('__module__')}.{properties.get('__qualname__')}"
+        )
         all_fields = dict()
         # mapping field names to base classes
         field_to_cls_hash = dict()
@@ -107,6 +110,10 @@ class SerializableMeta(type):
         slots.update(properties_field_slot_names)
 
         properties = properties_without_fields
+
+        # todo remove this prop when all versions below v1.0.0rc1 is eliminated
+        properties["_LEGACY_NAME_HASH"] = legacy_name_hash
+
         properties["_NAME_HASH"] = name_hash
         properties["_FIELDS"] = all_fields
         properties["_FIELD_ORDER"] = field_order
@@ -210,8 +217,8 @@ class SerializableSerializer(Serializer):
     """
 
     @classmethod
-    def _get_obj_field_count_key(cls, obj: Serializable):
-        return f"FC_{obj._NAME_HASH}"
+    def _get_obj_field_count_key(cls, obj: Serializable, legacy: bool = False):
+        return f"FC_{obj._NAME_HASH if not legacy else obj._LEGACY_NAME_HASH}"
 
     @classmethod
     def _get_field_values(cls, obj: Serializable, fields):
@@ -290,6 +297,12 @@ class SerializableSerializer(Serializer):
             server_cls_to_field_count = obj_class._CLS_TO_NON_PRIMITIVE_FIELD_COUNT
             server_fields = obj_class._NON_PRIMITIVE_FIELDS
 
+        legacy_to_new_hash = {
+            c._LEGACY_NAME_HASH: c._NAME_HASH
+            for c in obj_class.__mro__
+            if hasattr(c, "_NAME_HASH") and c._LEGACY_NAME_HASH != c._NAME_HASH
+        }
+
         if client_cls_to_field_count:
             field_num, server_field_num = 0, 0
             for cls_hash, count in client_cls_to_field_count.items():
@@ -301,20 +314,40 @@ class SerializableSerializer(Serializer):
                     if not is_primitive or value != {}:
                         cls._set_field_value(obj, field, value)
                 field_num += count
-                server_field_num += server_cls_to_field_count[cls_hash]
+                try:
+                    server_field_num += server_cls_to_field_count[cls_hash]
+                except KeyError:
+                    try:
+                        # todo remove this fallback when all
+                        #  versions below v1.0.0rc1 is eliminated
+                        server_field_num += server_cls_to_field_count[
+                            legacy_to_new_hash[cls_hash]
+                        ]
+                    except KeyError:
+                        # it is possible that certain type of field does not exist
+                        #  at server side
+                        pass
         else:
+            # handle legacy serialization style, with all fields sorted by name
             # todo remove this branch when all versions below v0.1.0b5 is eliminated
             from .field import AnyField
 
-            # legacy serialization style, with all fields sorted by name
             if is_primitive:
-                field_attr = "_legacy_deprecated_primitives"
+                new_field_attr = "_legacy_new_primitives"
+                deprecated_field_attr = "_legacy_deprecated_primitives"
             else:
-                field_attr = "_legacy_deprecated_non_primitives"
+                new_field_attr = "_legacy_new_non_primitives"
+                deprecated_field_attr = "_legacy_deprecated_non_primitives"
+
+            # remove fields added on later releases
+            new_names = set(getattr(obj_class, new_field_attr, None) or [])
+            server_fields = [f for f in server_fields if f.name not in new_names]
+
+            # fill fields deprecated on later releases
             deprecated_fields = []
             deprecated_names = set()
-            if hasattr(obj_class, field_attr):
-                deprecated_names = set(getattr(obj_class, field_attr))
+            if hasattr(obj_class, deprecated_field_attr):
+                deprecated_names = set(getattr(obj_class, deprecated_field_attr))
                 for field_name in deprecated_names:
                     field = AnyField(tag=field_name)
                     field.name = field_name
@@ -342,6 +375,12 @@ class SerializableSerializer(Serializer):
         field_count_data = self.get_public_data(
             context, self._get_obj_field_count_key(obj)
         )
+        if field_count_data is None:
+            # todo remove this fallback when all
+            #  versions below v1.0.0rc1 is eliminated
+            field_count_data = self.get_public_data(
+                context, self._get_obj_field_count_key(obj, legacy=True)
+            )
         if field_count_data is not None:
             cls_to_prim_key, cls_to_non_prim_key = msgpack.loads(field_count_data)
             cls_to_prim_key = dict(cls_to_prim_key)

maxframe/tensor/__init__.py

@@ -180,4 +180,63 @@ from .reduction import std, sum, var
 from .reshape import reshape
 from .ufunc import ufunc
 
+# isort: off
+# noinspection PyUnresolvedReferences
+from numpy import (
+    NAN,
+    NINF,
+    AxisError,
+    Inf,
+    NaN,
+    e,
+    errstate,
+    geterr,
+    inf,
+    nan,
+    newaxis,
+    pi,
+    seterr,
+)
+
+# import numpy types
+# noinspection PyUnresolvedReferences
+from numpy import (
+    bool_ as bool,
+    bytes_,
+    cfloat,
+    character,
+    complex64,
+    complex128,
+    complexfloating,
+    datetime64,
+    double,
+    dtype,
+    flexible,
+    float16,
+    float32,
+    float64,
+    floating,
+    generic,
+    inexact,
+    int8,
+    int16,
+    int32,
+    int64,
+    intc,
+    intp,
+    number,
+    integer,
+    object_ as object,
+    signedinteger,
+    timedelta64,
+    uint,
+    uint8,
+    uint16,
+    uint32,
+    uint64,
+    unicode_,
+    unsignedinteger,
+    void,
+)
+
 del fetch, ufunc

maxframe/tensor/base/unique.py

@@ -15,7 +15,7 @@
 
 import numpy as np
 
-from ... import opcodes as OperandDef
+from ... import opcodes
 from ...serialization.serializables import BoolField, Int32Field
 from ..core import TensorOrder
 from ..operators import TensorHasInput, TensorOperatorMixin
@@ -23,7 +23,7 @@ from ..utils import validate_axis
 
 
 class TensorUnique(TensorHasInput, TensorOperatorMixin):
-    _op_type_ = OperandDef.UNIQUE
+    _op_type_ = opcodes.UNIQUE
 
     return_index = BoolField("return_index", default=False)
     return_inverse = BoolField("return_inverse", default=False)

maxframe/tensor/statistics/quantile.py

@@ -16,7 +16,7 @@ from collections.abc import Iterable
 
 import numpy as np
 
-from ... import opcodes as OperandDef
+from ... import opcodes
 from ...core import ENTITY_TYPE
 from ...serialization.serializables import AnyField, BoolField, KeyField, StringField
 from ..core import TENSOR_TYPE, TensorOrder
@@ -43,7 +43,7 @@ q_error_msg = "Quantiles must be in the range [0, 1]"
 
 class TensorQuantile(TensorOperator, TensorOperatorMixin):
     __slots__ = ("q_error_msg",)
-    _op_type_ = OperandDef.QUANTILE
+    _op_type_ = opcodes.QUANTILE
 
     a = KeyField("a")
     q = AnyField("q")

maxframe/tests/utils.py

@@ -14,6 +14,7 @@
 
 import asyncio
 import functools
+import hashlib
 import os
 import queue
 import socket
@@ -25,7 +26,7 @@ import pytest
 from tornado import netutil
 
 from ..core import Tileable, TileableGraph
-from ..utils import create_event, lazy_import
+from ..utils import create_sync_primitive, lazy_import, to_binary
 
 try:
     from flaky import flaky
@@ -102,7 +103,7 @@ def run_app_in_thread(app_func):
     def fixture_func(*args, **kwargs):
         app_loop = asyncio.new_event_loop()
         q = queue.Queue()
-        exit_event = create_event(app_loop)
+        exit_event = create_sync_primitive(asyncio.Event, app_loop)
         app_thread = Thread(
             name="TestAppThread",
             target=app_thread_func,
@@ -162,3 +163,11 @@ def require_hadoop(func):
         not os.environ.get("WITH_HADOOP"), reason="Only run when hadoop is installed"
     )(func)
     return func
+
+
+def get_test_unique_name(size=None):
+    test_name = os.getenv("PYTEST_CURRENT_TEST", "pyodps_test")
+    digest = hashlib.md5(to_binary(test_name)).hexdigest()
+    if size:
+        digest = digest[:size]
+    return digest + "_" + str(os.getpid())

maxframe/utils.py

@@ -436,19 +436,27 @@ async def to_thread_pool(func, *args, pool=None, **kwargs):
     return await loop.run_in_executor(pool, func_call)
 
 
-def create_event(loop: asyncio.AbstractEventLoop) -> asyncio.Event:
+_PrimitiveType = TypeVar("_PrimitiveType")
+
+
+def create_sync_primitive(
+    cls: Type[_PrimitiveType], loop: asyncio.AbstractEventLoop
+) -> _PrimitiveType:
     """
-    Create an asyncio.Event in a certain event loop.
+    Create an asyncio sync primitive (locks, events, etc.)
+    in a certain event loop.
     """
-    if sys.version_info[1] < 10 or loop is None:
-        return asyncio.Event(loop=loop)
+    if sys.version_info[1] < 10:
+        return cls(loop=loop)
 
     # From Python3.10 the loop parameter has been removed. We should work around here.
-    old_loop = asyncio.get_running_loop()
-    asyncio.set_event_loop(loop)
-    event = asyncio.Event()
-    asyncio.set_event_loop(old_loop)
-    return event
+    old_loop = asyncio.get_event_loop()
+    try:
+        asyncio.set_event_loop(loop)
+        primitive = cls()
+    finally:
+        asyncio.set_event_loop(old_loop)
+    return primitive
 
 
 class ToThreadCancelledError(asyncio.CancelledError):

{maxframe-1.0.0rc1.dist-info → maxframe-1.0.0rc2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maxframe
-Version: 1.0.0rc1
+Version: 1.0.0rc2
 Summary: MaxFrame operator-based data analyze framework
 Requires-Dist: numpy <2.0.0,>=1.19.0
 Requires-Dist: pandas >=1.0.0
@@ -102,3 +102,76 @@ License
 
 Licensed under the `Apache License
 2.0 <https://www.apache.org/licenses/LICENSE-2.0.html>`__.
+MaxCompute MaxFrame Client
+==========================
+
+MaxFrame is a computational framework created by Alibaba Cloud to
+provide a way for Python developers to parallelize their code with
+MaxCompute. It creates a runnable computation graph locally, submits it
+to MaxCompute to execute and obtains results from MaxCompute.
+
+MaxFrame client is the client of MaxFrame. Currently it provides a
+DataFrame-based SDK with compatible APIs for pandas. In future, other
+common Python libraries like numpy and scikit-learn will be added as
+well. Python 3.7 is recommended for MaxFrame client to enable all
+functionalities while supports for higher Python versions are on the
+way.
+
+Installation
+------------
+
+You may install MaxFrame client through PIP:
+
+.. code:: bash
+
+   pip install maxframe
+
+Latest beta version can be installed with ``--pre`` argument:
+
+.. code:: bash
+
+   pip install --pre maxframe
+
+You can also install MaxFrame client from source code:
+
+.. code:: bash
+
+   pip install git+https://github.com/aliyun/alibabacloud-odps-maxframe-client.git
+
+Getting started
+---------------
+
+We show a simple code example of MaxFrame client which read data from a
+MaxCompute table, performs some simple data transform and writes back
+into MaxCompute.
+
+.. code:: python
+
+   import maxframe.dataframe as md
+   import os
+   from maxframe import new_session
+   from odps import ODPS
+
+   o = ODPS(
+       os.getenv('ALIBABA_CLOUD_ACCESS_KEY_ID'),
+       os.getenv('ALIBABA_CLOUD_ACCESS_KEY_SECRET'),
+       project='your-default-project',
+       endpoint='your-end-point',
+   )
+   session = new_session(o)
+
+   df = md.read_odps_table("source_table")
+   df["A"] = "prefix_" + df["A"]
+   md.to_odps_table(df, "prefix_source_table")
+
+Documentation
+-------------
+
+Detailed documentations can be found
+`here <https://maxframe.readthedocs.io>`__.
+
+License
+-------
+
+Licensed under the `Apache License
+2.0 <https://www.apache.org/licenses/LICENSE-2.0.html>`__.