maxframe 1.0.0rc4__cp39-cp39-win_amd64.whl → 1.1.0__cp39-cp39-win_amd64.whl

maxframe/learn/contrib/llm/multi_modal.py

@@ -0,0 +1,42 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import Any, Dict
+
+from ....dataframe.core import DATAFRAME_TYPE, SERIES_TYPE
+from .core import LLM
+
+
+class MultiModalLLM(LLM):
+    def generate(
+        self,
+        data,
+        prompt_template: Dict[str, Any],
+        params: Dict[str, Any] = None,
+    ):
+        raise NotImplementedError
+
+
+def generate(
+    data,
+    model: MultiModalLLM,
+    prompt_template: Dict[str, Any],
+    params: Dict[str, Any] = None,
+):
+    if not isinstance(data, DATAFRAME_TYPE) and not isinstance(data, SERIES_TYPE):
+        raise ValueError("data must be a maxframe dataframe or series object")
+    if not isinstance(model, MultiModalLLM):
+        raise ValueError("model must be a MultiModalLLM object")
+    params = params if params is not None else dict()
+    model.validate_params(params)
+    return model.generate(data, prompt_template, params)

maxframe/learn/contrib/llm/text.py

@@ -0,0 +1,42 @@
+# Copyright 1999-2024 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import Any, Dict
+
+from ....dataframe.core import DATAFRAME_TYPE, SERIES_TYPE
+from .core import LLM
+
+
+class TextLLM(LLM):
+    def generate(
+        self,
+        data,
+        prompt_template: Dict[str, Any],
+        params: Dict[str, Any] = None,
+    ):
+        raise NotImplementedError
+
+
+def generate(
+    data,
+    model: TextLLM,
+    prompt_template: Dict[str, Any],
+    params: Dict[str, Any] = None,
+):
+    if not isinstance(data, DATAFRAME_TYPE) and not isinstance(data, SERIES_TYPE):
+        raise ValueError("data must be a maxframe dataframe or series object")
+    if not isinstance(model, TextLLM):
+        raise ValueError("model must be a TextLLM object")
+    params = params if params is not None else dict()
+    model.validate_params(params)
+    return model.generate(data, prompt_template, params)

maxframe/lib/sparse/tests/test_sparse.py

@@ -55,13 +55,13 @@ def test_sparse_creation():
     s = SparseNDArray(s1_data)
     assert s.ndim == 2
     assert isinstance(s, SparseMatrix)
-    assert_array_equal(s.toarray(), s1_data.A)
-    assert_array_equal(s.todense(), s1_data.A)
+    assert_array_equal(s.toarray(), s1_data.toarray())
+    assert_array_equal(s.todense(), s1_data.toarray())
 
     ss = pickle.loads(pickle.dumps(s))
     assert s == ss
-    assert_array_equal(ss.toarray(), s1_data.A)
-    assert_array_equal(ss.todense(), s1_data.A)
+    assert_array_equal(ss.toarray(), s1_data.toarray())
+    assert_array_equal(ss.todense(), s1_data.toarray())
 
     v = SparseNDArray(v1, shape=(3,))
     assert s.ndim
@@ -331,12 +331,12 @@ def test_sparse_dot():
 
     assert_array_equal(mls.dot(s1, v1_s), s1.dot(v1_data))
     assert_array_equal(mls.dot(s2, v1_s), s2.dot(v1_data))
-    assert_array_equal(mls.dot(v2_s, s1), v2_data.dot(s1_data.A))
-    assert_array_equal(mls.dot(v2_s, s2), v2_data.dot(s2_data.A))
+    assert_array_equal(mls.dot(v2_s, s1), v2_data.dot(s1_data.toarray()))
+    assert_array_equal(mls.dot(v2_s, s2), v2_data.dot(s2_data.toarray()))
     assert_array_equal(mls.dot(v1_s, v1_s), v1_data.dot(v1_data), almost=True)
     assert_array_equal(mls.dot(v2_s, v2_s), v2_data.dot(v2_data), almost=True)
 
-    assert_array_equal(mls.dot(v2_s, s1, sparse=False), v2_data.dot(s1_data.A))
+    assert_array_equal(mls.dot(v2_s, s1, sparse=False), v2_data.dot(s1_data.toarray()))
     assert_array_equal(mls.dot(v1_s, v1_s, sparse=False), v1_data.dot(v1_data))
 
 
@@ -390,7 +390,7 @@ def test_sparse_fill_diagonal():
     arr = SparseNDArray(s1)
     arr.fill_diagonal(3)
 
-    expected = s1.copy().A
+    expected = s1.copy().toarray()
     np.fill_diagonal(expected, 3)
 
     np.testing.assert_array_equal(arr.toarray(), expected)
@@ -399,7 +399,7 @@ def test_sparse_fill_diagonal():
     arr = SparseNDArray(s1)
     arr.fill_diagonal(3, wrap=True)
 
-    expected = s1.copy().A
+    expected = s1.copy().toarray()
     np.fill_diagonal(expected, 3, wrap=True)
 
     np.testing.assert_array_equal(arr.toarray(), expected)
@@ -408,7 +408,7 @@ def test_sparse_fill_diagonal():
     arr = SparseNDArray(s1)
     arr.fill_diagonal([1, 2, 3])
 
-    expected = s1.copy().A
+    expected = s1.copy().toarray()
     np.fill_diagonal(expected, [1, 2, 3])
 
     np.testing.assert_array_equal(arr.toarray(), expected)
@@ -417,7 +417,7 @@ def test_sparse_fill_diagonal():
     arr = SparseNDArray(s1)
     arr.fill_diagonal([1, 2, 3], wrap=True)
 
-    expected = s1.copy().A
+    expected = s1.copy().toarray()
     np.fill_diagonal(expected, [1, 2, 3], wrap=True)
 
     np.testing.assert_array_equal(arr.toarray(), expected)
@@ -427,7 +427,7 @@ def test_sparse_fill_diagonal():
     arr = SparseNDArray(s1)
     arr.fill_diagonal(val)
 
-    expected = s1.copy().A
+    expected = s1.copy().toarray()
     np.fill_diagonal(expected, val)
 
     np.testing.assert_array_equal(arr.toarray(), expected)
@@ -437,7 +437,7 @@ def test_sparse_fill_diagonal():
     arr = SparseNDArray(s1)
     arr.fill_diagonal(val, wrap=True)
 
-    expected = s1.copy().A
+    expected = s1.copy().toarray()
     np.fill_diagonal(expected, val, wrap=True)
 
     np.testing.assert_array_equal(arr.toarray(), expected)
@@ -447,7 +447,7 @@ def test_sparse_fill_diagonal():
     arr = SparseNDArray(s1)
     arr.fill_diagonal(val)
 
-    expected = s1.copy().A
+    expected = s1.copy().toarray()
     np.fill_diagonal(expected, val)
 
     np.testing.assert_array_equal(arr.toarray(), expected)
@@ -457,7 +457,7 @@ def test_sparse_fill_diagonal():
     arr = SparseNDArray(s1)
     arr.fill_diagonal(val, wrap=True)
 
-    expected = s1.copy().A
+    expected = s1.copy().toarray()
     np.fill_diagonal(expected, val, wrap=True)
 
     np.testing.assert_array_equal(arr.toarray(), expected)

maxframe/opcodes.py

@@ -270,6 +270,7 @@ KURTOSIS = 351
 SEM = 352
 STR_CONCAT = 353
 MAD = 354
+MEDIAN = 355
 
 # tensor operator
 RESHAPE = 401
@@ -377,7 +378,6 @@ DROP_DUPLICATES = 728
 MELT = 729
 RENAME = 731
 INSERT = 732
-MAP_CHUNK = 733
 CARTESIAN_CHUNK = 734
 EXPLODE = 735
 REPLACE = 736
@@ -392,6 +392,10 @@ PIVOT_TABLE = 744
 
 FUSE = 801
 
+# LLM
+DASHSCOPE_TEXT_GENERATION = 810
+DASHSCOPE_MULTI_MODAL_GENERATION = 811
+
 # table like input for tensor
 TABLE_COO = 1003
 # store tensor as coo format
@@ -569,6 +573,8 @@ CHOLESKY_FUSE = 999988
 # MaxFrame-dedicated functions
 DATAFRAME_RESHUFFLE = 10001
 FLATMAP = 10002
+FLATJSON = 10003
+APPLY_CHUNK = 10004
 
 # MaxFrame internal operators
 DATAFRAME_PROJECTION_SAME_INDEX_MERGE = 100001

maxframe/serialization/core.pyx

@@ -37,7 +37,7 @@ from .._utils import NamedType
 from .._utils cimport TypeDispatcher
 
 from ..lib import wrapped_pickle as pickle
-from ..utils import arrow_type_from_str
+from ..utils import NoDefault, arrow_type_from_str, no_default
 
 try:
     from pandas import ArrowDtype
@@ -94,6 +94,7 @@ cdef:
     int COMPLEX_SERIALIZER = 12
     int SLICE_SERIALIZER = 13
     int REGEX_SERIALIZER = 14
+    int NO_DEFAULT_SERIALIZER = 15
     int PLACEHOLDER_SERIALIZER = 4096
 
 
@@ -803,6 +804,16 @@ cdef class RegexSerializer(Serializer):
         return re.compile((<bytes>(subs[0])).decode(), serialized[0])
 
 
+cdef class NoDefaultSerializer(Serializer):
+    serializer_id = NO_DEFAULT_SERIALIZER
+
+    cpdef serial(self, object obj, dict context):
+        return [], [], True
+
+    cpdef deserial(self, list obj, dict context, list subs):
+        return no_default
+
+
 cdef class Placeholder:
     """
     Placeholder object to reduce duplicated serialization
@@ -857,6 +868,7 @@ DtypeSerializer.register(ExtensionDtype)
 ComplexSerializer.register(complex)
 SliceSerializer.register(slice)
 RegexSerializer.register(re.Pattern)
+NoDefaultSerializer.register(NoDefault)
 PlaceholderSerializer.register(Placeholder)
 
 

maxframe/serialization/pandas.py

@@ -134,8 +134,10 @@ class ArraySerializer(Serializer):
                 data_parts = [obj.tolist()]
             else:
                 data_parts = [obj.to_numpy().tolist()]
-        else:
+        elif hasattr(obj, "_data"):
             data_parts = [getattr(obj, "_data")]
+        else:
+            data_parts = [getattr(obj, "_pa_array")]
         return [ser_type], [dtype] + data_parts, False
 
     def deserial(self, serialized: List, context: Dict, subs: List):
@@ -155,38 +157,66 @@ class PdTimestampSerializer(Serializer):
         else:
             zone_info = []
             ts = obj.to_pydatetime().timestamp()
-        return (
-            [int(ts), obj.microsecond, obj.nanosecond],
-            zone_info,
-            bool(zone_info),
-        )
+        elements = [int(ts), obj.microsecond, obj.nanosecond]
+        if hasattr(obj, "unit"):
+            elements.append(str(obj.unit))
+        return elements, zone_info, bool(zone_info)
 
     def deserial(self, serialized: List, context: Dict, subs: List):
         if subs:
-            val = pd.Timestamp.utcfromtimestamp(serialized[0]).replace(
-                microsecond=serialized[1], nanosecond=serialized[2]
-            )
-            val = val.replace(tzinfo=datetime.timezone.utc).tz_convert(subs[0])
+            pydt = datetime.datetime.utcfromtimestamp(serialized[0])
+            kwargs = {
+                "year": pydt.year,
+                "month": pydt.month,
+                "day": pydt.day,
+                "hour": pydt.hour,
+                "minute": pydt.minute,
+                "second": pydt.second,
+                "microsecond": serialized[1],
+                "nanosecond": serialized[2],
+                "tzinfo": datetime.timezone.utc,
+            }
+            if len(serialized) > 3:
+                kwargs["unit"] = serialized[3]
+            val = pd.Timestamp(**kwargs).tz_convert(subs[0])
         else:
-            val = pd.Timestamp.fromtimestamp(serialized[0]).replace(
-                microsecond=serialized[1], nanosecond=serialized[2]
-            )
+            pydt = datetime.datetime.fromtimestamp(serialized[0])
+            kwargs = {
+                "year": pydt.year,
+                "month": pydt.month,
+                "day": pydt.day,
+                "hour": pydt.hour,
+                "minute": pydt.minute,
+                "second": pydt.second,
+                "microsecond": serialized[1],
+                "nanosecond": serialized[2],
+            }
+            if len(serialized) >= 4:
+                kwargs["unit"] = serialized[3]
+            val = pd.Timestamp(**kwargs)
         return val
 
 
 class PdTimedeltaSerializer(Serializer):
     def serial(self, obj: pd.Timedelta, context: Dict):
-        return [int(obj.seconds), obj.microseconds, obj.nanoseconds, obj.days], [], True
+        elements = [int(obj.seconds), obj.microseconds, obj.nanoseconds, obj.days]
+        if hasattr(obj, "unit"):
+            elements.append(str(obj.unit))
+        return elements, [], True
 
     def deserial(self, serialized: List, context: Dict, subs: List):
         days = 0 if len(serialized) < 4 else serialized[3]
+        unit = None if len(serialized) < 5 else serialized[4]
         seconds, microseconds, nanoseconds = serialized[:3]
-        return pd.Timedelta(
-            days=days,
-            seconds=seconds,
-            microseconds=microseconds,
-            nanoseconds=nanoseconds,
-        )
+        kwargs = {
+            "days": days,
+            "seconds": seconds,
+            "microseconds": microseconds,
+            "nanoseconds": nanoseconds,
+        }
+        if unit is not None:
+            kwargs["unit"] = unit
+        return pd.Timedelta(**kwargs)
 
 
 class NoDefaultSerializer(Serializer):

maxframe/serialization/serializables/core.py

@@ -19,6 +19,7 @@ from typing import Any, Dict, List, Optional, Tuple, Type
 import msgpack
 
 from ...lib.mmh3 import hash
+from ...utils import no_default
 from ..core import Placeholder, Serializer, buffered, load_type
 from .field import Field
 from .field_type import DictType, ListType, PrimitiveFieldType, TupleType
@@ -211,6 +212,22 @@ class _NoFieldValue:
 _no_field_value = _NoFieldValue()
 
 
+def _to_primitive_placeholder(v: Any) -> Any:
+    if v is _no_field_value or v is no_default:
+        return {}
+    return v
+
+
+def _restore_primitive_placeholder(v: Any) -> Any:
+    if type(v) is dict:
+        if v == {}:
+            return _no_field_value
+        else:
+            return v
+    else:
+        return v
+
+
 class SerializableSerializer(Serializer):
     """
     Leverage DictSerializer to perform serde.
@@ -241,9 +258,7 @@ class SerializableSerializer(Serializer):
         else:
             primitive_vals = self._get_field_values(obj, obj._PRIMITIVE_FIELDS)
             # replace _no_field_value as {} to make them msgpack-serializable
-            primitive_vals = [
-                v if v is not _no_field_value else {} for v in primitive_vals
-            ]
+            primitive_vals = [_to_primitive_placeholder(v) for v in primitive_vals]
             if obj._cache_primitive_serial:
                 primitive_vals = msgpack.dumps(primitive_vals)
                 _primitive_serial_cache[obj] = primitive_vals
@@ -311,7 +326,9 @@ class SerializableSerializer(Serializer):
                 cls_fields = server_fields[server_field_num : field_num + count]
                 cls_values = values[field_num : field_num + count]
                 for field, value in zip(cls_fields, cls_values):
-                    if not is_primitive or value != {}:
+                    if is_primitive:
+                        value = _restore_primitive_placeholder(value)
+                    if not is_primitive or value is not _no_field_value:
                         cls._set_field_value(obj, field, value)
                 field_num += count
                 try:
@@ -356,7 +373,9 @@ class SerializableSerializer(Serializer):
                 server_fields + deprecated_fields, key=lambda f: f.name
             )
             for field, value in zip(server_fields, values):
-                if not is_primitive or value != {}:
+                if is_primitive:
+                    value = _restore_primitive_placeholder(value)
+                if not is_primitive or value is not _no_field_value:
                     try:
                         cls._set_field_value(obj, field, value)
                     except AttributeError:  # pragma: no cover

maxframe/serialization/serializables/field_type.py

@@ -46,6 +46,9 @@ class PrimitiveType(Enum):
     complex128 = 25
 
 
+_np_unicode = np.unicode_ if hasattr(np, "unicode_") else np.str_
+
+
 _primitive_type_to_valid_types = {
     PrimitiveType.bool: (bool, np.bool_),
     PrimitiveType.int8: (int, np.int8),
@@ -60,7 +63,7 @@ _primitive_type_to_valid_types = {
     PrimitiveType.float32: (float, np.float32),
     PrimitiveType.float64: (float, np.float64),
     PrimitiveType.bytes: (bytes, np.bytes_),
-    PrimitiveType.string: (str, np.unicode_),
+    PrimitiveType.string: (str, _np_unicode),
     PrimitiveType.complex64: (complex, np.complex64),
     PrimitiveType.complex128: (complex, np.complex128),
 }

maxframe/serialization/serializables/tests/test_serializable.py

@@ -21,6 +21,7 @@ import pytest
 
 from ....core import EntityData
 from ....lib.wrapped_pickle import switch_unpickle
+from ....utils import no_default
 from ... import deserialize, serialize
 from .. import (
     AnyField,
@@ -143,6 +144,7 @@ class MySerializable(Serializable):
         oneof1_val=f"{__name__}.MySerializable",
         oneof2_val=MySimpleSerializable,
     )
+    _no_default_val = Float64Field("no_default_val", default=no_default)
 
 
 @pytest.mark.parametrize("set_is_ci", [False, True], indirect=True)
@@ -187,6 +189,7 @@ def test_serializable(set_is_ci):
         _dict_val={"a": b"bytes_value"},
         _ref_val=MySerializable(),
         _oneof_val=MySerializable(_id="2"),
+        _no_default_val=no_default,
     )
 
     header, buffers = serialize(my_serializable)
@@ -234,7 +237,11 @@ def _assert_serializable_eq(my_serializable, my_serializable2):
         if not hasattr(my_serializable, field.name):
             continue
         expect_value = getattr(my_serializable, field_name)
-        actual_value = getattr(my_serializable2, field_name)
+        if expect_value is no_default:
+            assert not hasattr(my_serializable2, field.name)
+            continue
+        else:
+            actual_value = getattr(my_serializable2, field_name)
         if isinstance(expect_value, np.ndarray):
             np.testing.assert_array_equal(expect_value, actual_value)
         elif isinstance(expect_value, pd.DataFrame):

maxframe/serialization/tests/test_serial.py

@@ -42,7 +42,7 @@ except ImportError:
 from ...lib.sparse import SparseMatrix
 from ...lib.wrapped_pickle import switch_unpickle
 from ...tests.utils import require_cudf, require_cupy
-from ...utils import lazy_import
+from ...utils import lazy_import, no_default
 from .. import (
     PickleContainer,
     RemoteException,
@@ -90,6 +90,7 @@ class CustomNamedTuple(NamedTuple):
         pd.Timedelta(102.234154131),
         {"abc": 5.6, "def": [3.4], "gh": None, "ijk": {}},
         OrderedDict([("abcd", 5.6)]),
+        no_default,
     ],
 )
 @switch_unpickle

maxframe/tensor/__init__.py

@@ -191,11 +191,6 @@ from .ufunc import ufunc
 # isort: off
 # noinspection PyUnresolvedReferences
 from numpy import (
-    NAN,
-    NINF,
-    AxisError,
-    Inf,
-    NaN,
     e,
     errstate,
     geterr,
@@ -206,12 +201,21 @@ from numpy import (
     seterr,
 )
 
+try:
+    from numpy.exceptions import AxisError
+except ImportError:
+    from numpy import AxisError
+
+NAN = nan
+NINF = -inf
+Inf = inf
+NaN = nan
+
 # import numpy types
 # noinspection PyUnresolvedReferences
 from numpy import (
     bool_ as bool,
     bytes_,
-    cfloat,
     character,
     complex64,
     complex128,
@@ -242,9 +246,17 @@ from numpy import (
     uint16,
     uint32,
     uint64,
-    unicode_,
     unsignedinteger,
     void,
 )
 
+try:
+    from numpy import cfloat
+except ImportError:
+    from numpy import cdouble as cfloat
+try:
+    from numpy import str_ as unicode_
+except ImportError:
+    from numpy import unicode_
+
 del fetch, ufunc

maxframe/tests/utils.py

@@ -18,11 +18,13 @@ import hashlib
 import os
 import queue
 import socket
+import time
 import types
 from threading import Thread
 from typing import Dict, List, Optional, Set, Tuple
 
 import pytest
+from odps import ODPS
 from tornado import netutil
 
 from ..core import Tileable, TileableGraph
@@ -171,3 +173,17 @@ def get_test_unique_name(size=None):
     if size:
         digest = digest[:size]
     return digest + "_" + str(os.getpid())
+
+
+def assert_mf_index_dtype(idx_obj, dtype):
+    from ..dataframe.core import IndexValue
+
+    assert isinstance(idx_obj, IndexValue.IndexBase) and idx_obj.dtype == dtype
+
+
+def ensure_table_deleted(odps_entry: ODPS, table_name: str) -> None:
+    retry_times = 20
+    while odps_entry.exist_table(table_name) and retry_times > 0:
+        time.sleep(1)
+        retry_times -= 1
+    assert not odps_entry.exist_table(table_name)

maxframe/udf.py

@@ -19,6 +19,7 @@ from odps.models import Resource
 
 from .serialization.serializables import (
     BoolField,
+    DictField,
     FieldTypes,
     FunctionField,
     ListField,
@@ -54,6 +55,10 @@ class MarkedFunction(Serializable):
     func = FunctionField("func")
     resources = ListField("resources", FieldTypes.string, default_factory=list)
     pythonpacks = ListField("pythonpacks", FieldTypes.reference, default_factory=list)
+    expect_engine = StringField("expect_engine", default=None)
+    expect_resources = DictField(
+        "expect_resources", FieldTypes.string, default_factory=dict
+    )
 
     def __init__(self, func: Optional[Callable] = None, **kw):
         super().__init__(func=func, **kw)
@@ -120,6 +125,28 @@ def with_python_requirements(
     return func_wrapper
 
 
+def with_running_options(
+    *,
+    engine: Optional[str] = None,
+    cpu: Optional[int] = None,
+    memory: Optional[int] = None,
+    **kwargs,
+):
+    engine = engine.upper() if engine else None
+    resources = {"cpu": cpu, "memory": memory, **kwargs}
+
+    def func_wrapper(func):
+        if all(v is None for v in (engine, cpu, memory)):
+            return func
+        if isinstance(func, MarkedFunction):
+            func.expect_engine = engine
+            func.expect_resources = resources
+            return func
+        return MarkedFunction(func, expect_engine=engine, expect_resources=resources)
+
+    return func_wrapper
+
+
 with_resource_libraries = with_resources