maxframe 0.1.0b1__cp38-cp38-macosx_10_9_universal2.whl → 0.1.0b3__cp38-cp38-macosx_10_9_universal2.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of maxframe might be problematic. Click here for more details.
- maxframe/_utils.cpython-38-darwin.so +0 -0
- maxframe/codegen.py +88 -19
- maxframe/config/config.py +10 -0
- maxframe/core/entity/executable.py +1 -0
- maxframe/core/entity/objects.py +3 -2
- maxframe/core/graph/core.cpython-38-darwin.so +0 -0
- maxframe/core/graph/core.pyx +2 -2
- maxframe/core/operator/base.py +14 -0
- maxframe/dataframe/__init__.py +3 -1
- maxframe/dataframe/datasource/from_records.py +4 -0
- maxframe/dataframe/datasource/read_odps_query.py +295 -0
- maxframe/dataframe/datasource/read_odps_table.py +1 -1
- maxframe/dataframe/datasource/tests/test_datasource.py +84 -1
- maxframe/dataframe/groupby/__init__.py +4 -0
- maxframe/dataframe/groupby/core.py +5 -0
- maxframe/dataframe/misc/to_numeric.py +4 -0
- maxframe/dataframe/window/aggregation.py +1 -24
- maxframe/dataframe/window/ewm.py +0 -7
- maxframe/dataframe/window/tests/test_ewm.py +0 -6
- maxframe/errors.py +21 -0
- maxframe/lib/aio/isolation.py +6 -1
- maxframe/lib/mmh3.cpython-38-darwin.so +0 -0
- maxframe/opcodes.py +1 -0
- maxframe/protocol.py +25 -5
- maxframe/serialization/core.cpython-38-darwin.so +0 -0
- maxframe/serialization/exception.py +2 -1
- maxframe/serialization/serializables/core.py +6 -1
- maxframe/serialization/serializables/field.py +2 -0
- maxframe/tensor/core.py +3 -3
- maxframe/tests/test_codegen.py +69 -0
- maxframe/tests/test_protocol.py +16 -8
- maxframe/tests/utils.py +1 -0
- maxframe/udf.py +15 -16
- maxframe/utils.py +21 -1
- {maxframe-0.1.0b1.dist-info → maxframe-0.1.0b3.dist-info}/METADATA +1 -74
- {maxframe-0.1.0b1.dist-info → maxframe-0.1.0b3.dist-info}/RECORD +42 -39
- {maxframe-0.1.0b1.dist-info → maxframe-0.1.0b3.dist-info}/WHEEL +1 -1
- maxframe_client/clients/framedriver.py +7 -7
- maxframe_client/session/task.py +31 -3
- maxframe_client/session/tests/test_task.py +29 -11
- maxframe_client/tests/test_session.py +2 -0
- {maxframe-0.1.0b1.dist-info → maxframe-0.1.0b3.dist-info}/top_level.txt +0 -0
maxframe/udf.py
CHANGED
|
@@ -26,9 +26,7 @@ from .serialization.serializables import (
|
|
|
26
26
|
|
|
27
27
|
class MarkedFunction(Serializable):
|
|
28
28
|
func = FunctionField("func")
|
|
29
|
-
|
|
30
|
-
"resource_libraries", FieldTypes.string, default_factory=list
|
|
31
|
-
)
|
|
29
|
+
resources = ListField("resources", FieldTypes.string, default_factory=list)
|
|
32
30
|
|
|
33
31
|
def __init__(self, func: Optional[Callable] = None, **kw):
|
|
34
32
|
super().__init__(func=func, **kw)
|
|
@@ -43,33 +41,34 @@ class MarkedFunction(Serializable):
|
|
|
43
41
|
return f"<MarkedFunction {self.func!r}>"
|
|
44
42
|
|
|
45
43
|
|
|
46
|
-
def
|
|
47
|
-
*resources: Union[str, Resource], use_wrapper_class: bool = True
|
|
48
|
-
):
|
|
44
|
+
def with_resources(*resources: Union[str, Resource], use_wrapper_class: bool = True):
|
|
49
45
|
def res_to_str(res: Union[str, Resource]) -> str:
|
|
50
46
|
if isinstance(res, str):
|
|
51
47
|
return res
|
|
52
|
-
res_parts = [res.project]
|
|
48
|
+
res_parts = [res.project.name]
|
|
53
49
|
if res.schema:
|
|
54
|
-
res_parts.
|
|
55
|
-
res_parts.
|
|
56
|
-
return "
|
|
50
|
+
res_parts.extend(["schemas", res.schema])
|
|
51
|
+
res_parts.extend(["resources", res.name])
|
|
52
|
+
return "/".join(res_parts)
|
|
57
53
|
|
|
58
54
|
def func_wrapper(func):
|
|
59
55
|
str_resources = [res_to_str(r) for r in resources]
|
|
60
56
|
if not use_wrapper_class:
|
|
61
|
-
func.
|
|
57
|
+
func.resources = str_resources
|
|
62
58
|
return func
|
|
63
59
|
|
|
64
60
|
if isinstance(func, MarkedFunction):
|
|
65
|
-
func.
|
|
61
|
+
func.resources = str_resources
|
|
66
62
|
return func
|
|
67
|
-
return MarkedFunction(func,
|
|
63
|
+
return MarkedFunction(func, resources=list(str_resources))
|
|
68
64
|
|
|
69
65
|
return func_wrapper
|
|
70
66
|
|
|
71
67
|
|
|
72
|
-
|
|
68
|
+
with_resource_libraries = with_resources
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def get_udf_resources(
|
|
73
72
|
func: Callable,
|
|
74
|
-
) ->
|
|
75
|
-
return getattr(func, "
|
|
73
|
+
) -> List[Union[Resource, str]]:
|
|
74
|
+
return getattr(func, "resources", None) or []
|
maxframe/utils.py
CHANGED
|
@@ -338,6 +338,14 @@ def deserialize_serializable(ser_serializable: bytes):
|
|
|
338
338
|
return deserialize(header2, buffers2)
|
|
339
339
|
|
|
340
340
|
|
|
341
|
+
def skip_na_call(func: Callable):
|
|
342
|
+
@functools.wraps(func)
|
|
343
|
+
def new_func(x):
|
|
344
|
+
return func(x) if x is not None else None
|
|
345
|
+
|
|
346
|
+
return new_func
|
|
347
|
+
|
|
348
|
+
|
|
341
349
|
def url_path_join(*pieces):
|
|
342
350
|
"""Join components of url into a relative url
|
|
343
351
|
|
|
@@ -450,6 +458,9 @@ _ToThreadRetType = TypeVar("_ToThreadRetType")
|
|
|
450
458
|
|
|
451
459
|
|
|
452
460
|
class ToThreadMixin:
|
|
461
|
+
_thread_pool_size = 1
|
|
462
|
+
_counter = itertools.count().__next__
|
|
463
|
+
|
|
453
464
|
def __del__(self):
|
|
454
465
|
if hasattr(self, "_pool"):
|
|
455
466
|
kw = {"wait": False}
|
|
@@ -466,7 +477,10 @@ class ToThreadMixin:
|
|
|
466
477
|
**kwargs,
|
|
467
478
|
) -> _ToThreadRetType:
|
|
468
479
|
if not hasattr(self, "_pool"):
|
|
469
|
-
self._pool = concurrent.futures.ThreadPoolExecutor(
|
|
480
|
+
self._pool = concurrent.futures.ThreadPoolExecutor(
|
|
481
|
+
self._thread_pool_size,
|
|
482
|
+
thread_name_prefix=f"{type(self).__name__}Pool-{self._counter()}",
|
|
483
|
+
)
|
|
470
484
|
|
|
471
485
|
task = asyncio.create_task(
|
|
472
486
|
to_thread_pool(func, *args, **kwargs, pool=self._pool)
|
|
@@ -1081,3 +1095,9 @@ def arrow_type_from_str(type_str: str) -> pa.DataType:
|
|
|
1081
1095
|
if len(value_stack) > 1:
|
|
1082
1096
|
raise ValueError(f"Cannot parse type {type_str}")
|
|
1083
1097
|
return value_stack[-1]
|
|
1098
|
+
|
|
1099
|
+
|
|
1100
|
+
def get_python_tag():
|
|
1101
|
+
# todo add implementation suffix for non-GIL tags when PEP703 is ready
|
|
1102
|
+
version_info = sys.version_info
|
|
1103
|
+
return f"cp{version_info[0]}{version_info[1]}"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: maxframe
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.0b3
|
|
4
4
|
Summary: MaxFrame operator-based data analyze framework
|
|
5
5
|
Requires-Dist: numpy >=1.19.0
|
|
6
6
|
Requires-Dist: pandas >=1.0.0
|
|
@@ -102,76 +102,3 @@ License
|
|
|
102
102
|
|
|
103
103
|
Licensed under the `Apache License
|
|
104
104
|
2.0 <https://www.apache.org/licenses/LICENSE-2.0.html>`__.
|
|
105
|
-
MaxCompute MaxFrame Client
|
|
106
|
-
==========================
|
|
107
|
-
|
|
108
|
-
MaxFrame is a computational framework created by Alibaba Cloud to
|
|
109
|
-
provide a way for Python developers to parallelize their code with
|
|
110
|
-
MaxCompute. It creates a runnable computation graph locally, submits it
|
|
111
|
-
to MaxCompute to execute and obtains results from MaxCompute.
|
|
112
|
-
|
|
113
|
-
MaxFrame client is the client of MaxFrame. Currently it provides a
|
|
114
|
-
DataFrame-based SDK with compatible APIs for pandas. In future, other
|
|
115
|
-
common Python libraries like numpy and scikit-learn will be added as
|
|
116
|
-
well. Python 3.7 is recommended for MaxFrame client to enable all
|
|
117
|
-
functionalities while supports for higher Python versions are on the
|
|
118
|
-
way.
|
|
119
|
-
|
|
120
|
-
Installation
|
|
121
|
-
------------
|
|
122
|
-
|
|
123
|
-
You may install MaxFrame client through PIP:
|
|
124
|
-
|
|
125
|
-
.. code:: bash
|
|
126
|
-
|
|
127
|
-
pip install maxframe
|
|
128
|
-
|
|
129
|
-
Latest beta version can be installed with ``--pre`` argument:
|
|
130
|
-
|
|
131
|
-
.. code:: bash
|
|
132
|
-
|
|
133
|
-
pip install --pre maxframe
|
|
134
|
-
|
|
135
|
-
You can also install MaxFrame client from source code:
|
|
136
|
-
|
|
137
|
-
.. code:: bash
|
|
138
|
-
|
|
139
|
-
pip install git+https://github.com/aliyun/aliyun-odps-maxframe.git
|
|
140
|
-
|
|
141
|
-
Getting started
|
|
142
|
-
---------------
|
|
143
|
-
|
|
144
|
-
We show a simple code example of MaxFrame client which read data from a
|
|
145
|
-
MaxCompute table, performs some simple data transform and writes back
|
|
146
|
-
into MaxCompute.
|
|
147
|
-
|
|
148
|
-
.. code:: python
|
|
149
|
-
|
|
150
|
-
import maxframe.dataframe as md
|
|
151
|
-
import os
|
|
152
|
-
from maxframe import new_session
|
|
153
|
-
from odps import ODPS
|
|
154
|
-
|
|
155
|
-
o = ODPS(
|
|
156
|
-
os.getenv('ALIBABA_CLOUD_ACCESS_KEY_ID'),
|
|
157
|
-
os.getenv('ALIBABA_CLOUD_ACCESS_KEY_SECRET'),
|
|
158
|
-
project='your-default-project',
|
|
159
|
-
endpoint='your-end-point',
|
|
160
|
-
)
|
|
161
|
-
session = new_session(o)
|
|
162
|
-
|
|
163
|
-
df = md.read_odps_table("source_table")
|
|
164
|
-
df["A"] = "prefix_" + df["A"]
|
|
165
|
-
md.to_odps_table(df, "prefix_source_table")
|
|
166
|
-
|
|
167
|
-
Documentation
|
|
168
|
-
-------------
|
|
169
|
-
|
|
170
|
-
Detailed documentations can be found
|
|
171
|
-
`here <https://maxframe.readthedocs.io>`__.
|
|
172
|
-
|
|
173
|
-
License
|
|
174
|
-
-------
|
|
175
|
-
|
|
176
|
-
Licensed under the `Apache License
|
|
177
|
-
2.0 <https://www.apache.org/licenses/LICENSE-2.0.html>`__.
|
|
@@ -1,40 +1,41 @@
|
|
|
1
|
-
maxframe-0.1.
|
|
2
|
-
maxframe-0.1.
|
|
3
|
-
maxframe-0.1.
|
|
4
|
-
maxframe-0.1.
|
|
1
|
+
maxframe-0.1.0b3.dist-info/RECORD,,
|
|
2
|
+
maxframe-0.1.0b3.dist-info/WHEEL,sha256=J_5GGGAognyPkCA4mep3TG73WFTRAhUZ5YHiyekwc-A,113
|
|
3
|
+
maxframe-0.1.0b3.dist-info/top_level.txt,sha256=O_LOO6KS5Y1ZKdmCjA9mzfg0-b1sB_P6Oy-hD8aSDfM,25
|
|
4
|
+
maxframe-0.1.0b3.dist-info/METADATA,sha256=056zhE5bxFHo_1CAOnpi1A2Ov1JI8hg435HsygF02lA,3043
|
|
5
5
|
maxframe_client/conftest.py,sha256=7cwy2sFy5snEaxvtMvxfYFUnG6WtYC_9XxVrwJxOpcU,643
|
|
6
6
|
maxframe_client/__init__.py,sha256=3b-z0oFVVwtIzVFBxOb9pw7gz4IhTSh4FiHtVgnxS4Q,724
|
|
7
7
|
maxframe_client/fetcher.py,sha256=Ys_qu2qtniXuj9YSfeHvevdrAAEgm8k4YjyoZusdVmg,6813
|
|
8
8
|
maxframe_client/clients/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
|
|
9
9
|
maxframe_client/clients/spe.py,sha256=ArZMNQ7olicI4O1JO7CyRP7-hb60DF71ZKCTO0N39uE,3593
|
|
10
|
-
maxframe_client/clients/framedriver.py,sha256=
|
|
11
|
-
maxframe_client/tests/test_session.py,sha256=
|
|
10
|
+
maxframe_client/clients/framedriver.py,sha256=Rn09529D2qBTgNGc0oCY0l7b3FgzT87TqS1nujGQaHw,4463
|
|
11
|
+
maxframe_client/tests/test_session.py,sha256=75mxU5UTWjb1loQLo9BAkg7BzKuj4vYX49j4vP1-8fA,6307
|
|
12
12
|
maxframe_client/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
|
|
13
13
|
maxframe_client/tests/test_fetcher.py,sha256=q7kYCznM6WSxx9TCbHrxs7Zy1L2a5zu9D-Pi1XNgQzg,3516
|
|
14
|
-
maxframe_client/session/task.py,sha256=
|
|
14
|
+
maxframe_client/session/task.py,sha256=z5j8qtBM6cs_lZrvfy4Ji3F3sVOhPOCr5r1RsNe7rN4,11102
|
|
15
15
|
maxframe_client/session/graph.py,sha256=nwILNOIVaIf4E3xWffTAAlRsKRYU_zGW3oVO10du8Xw,4351
|
|
16
16
|
maxframe_client/session/__init__.py,sha256=KPqhSlAJiuUz8TC-z5o7mHDVXzLSqWwrZ33zNni7piY,832
|
|
17
17
|
maxframe_client/session/consts.py,sha256=R37BxDF3kgCy0qmDdwLaH5jB7mb7SzfYV6g9yHBKAwk,1344
|
|
18
18
|
maxframe_client/session/odps.py,sha256=uuPk5rc1OpFk9PNFU1R6r4HQXOHAOU7ZVvbcdGk_N_s,16248
|
|
19
|
-
maxframe_client/session/tests/test_task.py,sha256=
|
|
19
|
+
maxframe_client/session/tests/test_task.py,sha256=lDdw3gToaM3xSaRXEmHUoAo2h0id7t4v_VvpdKxQAao,3279
|
|
20
20
|
maxframe_client/session/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
|
|
21
21
|
maxframe/_utils.pyx,sha256=I4kmfhNr-xXK2ak22dr4Vwahzn-JmTaYctbL3y9_UBQ,17017
|
|
22
22
|
maxframe/conftest.py,sha256=ZzwKhGp7LAVpzQYJkniwIUQqIegcaDQAhyzDyU2qld4,4264
|
|
23
|
-
maxframe/opcodes.py,sha256=
|
|
23
|
+
maxframe/opcodes.py,sha256=L1_GGenexNnD4dk1ueelgpAebHCC-cY7d--1QPhw8l4,9984
|
|
24
24
|
maxframe/env.py,sha256=_K499f7giN7Iu9f39iI9p_naaEDoJ0rx8dInbzqFOVI,1402
|
|
25
25
|
maxframe/mixin.py,sha256=HBAeWYGb7N6ZIgkA-YpkKiSY1GetcEVNTuMb0ieznBs,3524
|
|
26
|
-
maxframe/protocol.py,sha256=
|
|
26
|
+
maxframe/protocol.py,sha256=LjjE6iw0ZVx82tBMbff4izkGuiJxRG0MTOaPYYpRL10,14190
|
|
27
27
|
maxframe/session.py,sha256=TspAq0EUhi-7VQb1kAuQyjhpX8vMYqgdk4vzbGCkJcY,35344
|
|
28
28
|
maxframe/__init__.py,sha256=YGnga-nYEppPoDZoZ5s64PZ0RYLaWtcYtmYSLTjKUBE,976
|
|
29
|
-
maxframe/_utils.cpython-38-darwin.so,sha256=
|
|
30
|
-
maxframe/utils.py,sha256=
|
|
29
|
+
maxframe/_utils.cpython-38-darwin.so,sha256=ytSSKdCXprt0R8d_Ohc4o1DcAvCMBNZA0n0pRjIRznY,914712
|
|
30
|
+
maxframe/utils.py,sha256=BrMJLO0-iPKABkNztXWpqiQWHoe-X75HGiCJr5aeles,33914
|
|
31
31
|
maxframe/extension.py,sha256=4IzF9sPyaRoAzLud0iDLooOgcyq4QunXH0ki3q9Hn8I,2374
|
|
32
|
-
maxframe/
|
|
32
|
+
maxframe/errors.py,sha256=nQZoLGdLJz-Uf9c2vUvQl08QMvRqtkBOhKfdPYRZA4o,690
|
|
33
|
+
maxframe/udf.py,sha256=tWOMTkNqGWw3ZpNC9wEU0GGNSBV8sV7E-Ye80DI28eg,2241
|
|
33
34
|
maxframe/typing_.py,sha256=fzHETru3IOZAJwU9I7n_ib3wHuQRJ9XFVmAk7WpqkMo,1096
|
|
34
|
-
maxframe/codegen.py,sha256=
|
|
35
|
+
maxframe/codegen.py,sha256=pSiGHoEo9YajFPHbFHvi7fGkiJmAQdBCe0mMXNOG6-U,15846
|
|
35
36
|
maxframe/_utils.pxd,sha256=AhJ4vA_UqZqPshi5nvIZq1xgr80fhIVQ9dm5-UdkYJ8,1154
|
|
36
37
|
maxframe/dataframe/arrays.py,sha256=RWzimUcrds5CsIlPausfJAkLUjcktBSSXwdXyUNKEtU,28752
|
|
37
|
-
maxframe/dataframe/__init__.py,sha256=
|
|
38
|
+
maxframe/dataframe/__init__.py,sha256=tpbt4OgW4hoU_1OsQu1WSxbpZD4YPXW_oT4xOxNPNqE,2112
|
|
38
39
|
maxframe/dataframe/core.py,sha256=mJ5I_qBP8HDAzlwPnkj8RN4EyBHC8d5unbe2LC6HKXg,73641
|
|
39
40
|
maxframe/dataframe/initializer.py,sha256=4BpZJB8bbyFnABUYWBrk_qzzrogEsWgFuU21Ma9IsjY,10264
|
|
40
41
|
maxframe/dataframe/utils.py,sha256=qWRo51rcMTlo4mvZ8ZZq1zIF9CiAgU1qRtoCAaYrR34,44111
|
|
@@ -47,7 +48,7 @@ maxframe/dataframe/statistics/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0
|
|
|
47
48
|
maxframe/dataframe/misc/accessor.py,sha256=QXjV5Q6LW0KNs67X5h2bCBhwHV8R7riP6TmTY_ECpmA,10027
|
|
48
49
|
maxframe/dataframe/misc/describe.py,sha256=Qvmv_5rj-fk_ABcS5h8IxNHY-EAPB1rG446LiB1MRgU,4387
|
|
49
50
|
maxframe/dataframe/misc/astype.py,sha256=OkGDO6PBZli46m0kKq0BOO2OOwJWEyatzeBFV_mnKz8,7797
|
|
50
|
-
maxframe/dataframe/misc/to_numeric.py,sha256=
|
|
51
|
+
maxframe/dataframe/misc/to_numeric.py,sha256=gG5fxEtC71SkDeHaJ0Vd1JZrIM8gMHGFaiwyFtMwbWs,6262
|
|
51
52
|
maxframe/dataframe/misc/check_monotonic.py,sha256=apHehWNRUTuzux3hI4JflWiuvpaeEFwtpPTWVide7wU,2422
|
|
52
53
|
maxframe/dataframe/misc/datetimes.py,sha256=fgLczaX-yDdairnNW1EsabEzoioP5gUsO0ukYANisLE,2503
|
|
53
54
|
maxframe/dataframe/misc/string_.py,sha256=eG4uMA6ScB0Pfby6pBLH24Ln01Aai2aAn-hMyRM5YFo,8101
|
|
@@ -82,13 +83,14 @@ maxframe/dataframe/datasource/from_index.py,sha256=2061zsQn-BhyHTT0X9tE0JK8vLxQU
|
|
|
82
83
|
maxframe/dataframe/datasource/dataframe.py,sha256=LxAKF4gBIHhnJQPuaAUdIEyMAq7HTfiEeNVls5n4I4A,2023
|
|
83
84
|
maxframe/dataframe/datasource/series.py,sha256=QcYiBNcR8jjH6vdO6l6H9F46KHmlBqVCTI2tv9eyZ9w,1909
|
|
84
85
|
maxframe/dataframe/datasource/__init__.py,sha256=C8EKsHTJi-1jvJUKIpZtMtsK-ZID3dtxL1voXnaltTs,640
|
|
86
|
+
maxframe/dataframe/datasource/read_odps_query.py,sha256=nj8l38S0iVAXgNXgzDFERO-HNp6lJ1GahImwDpEzXXw,9821
|
|
85
87
|
maxframe/dataframe/datasource/core.py,sha256=ozFmDgw1og7nK9_jU-u3tLEq9pNbitN-8w8XWdbKkJ0,2687
|
|
86
88
|
maxframe/dataframe/datasource/date_range.py,sha256=CDGpxDyjLwnb66j-MIiiTfXGXHGh5MLhEmj6x2riIlU,17244
|
|
87
|
-
maxframe/dataframe/datasource/read_odps_table.py,sha256=
|
|
89
|
+
maxframe/dataframe/datasource/read_odps_table.py,sha256=VCqWxJswcjujVoUNXb2kbTkOZroMkFCg5n272Yn7ME4,9045
|
|
88
90
|
maxframe/dataframe/datasource/read_parquet.py,sha256=9auOcy8snTxCOohgXZCUXfT_O39irdkBngZH5svgx0E,14531
|
|
89
91
|
maxframe/dataframe/datasource/from_tensor.py,sha256=4viuN5SLLye7Xeb8kouOpm-osoQ2yEovWTDNPQuW8gE,14727
|
|
90
|
-
maxframe/dataframe/datasource/from_records.py,sha256=
|
|
91
|
-
maxframe/dataframe/datasource/tests/test_datasource.py,sha256=
|
|
92
|
+
maxframe/dataframe/datasource/from_records.py,sha256=WBYouYyg7m_8NJdN-yUWSfJlIpm6DVP3IMfLXZFugyI,3442
|
|
93
|
+
maxframe/dataframe/datasource/tests/test_datasource.py,sha256=4O3N-XD-MpJxEQfILu4cS7gU82hqgS9g9gnDDEsw56k,14640
|
|
92
94
|
maxframe/dataframe/datasource/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
|
|
93
95
|
maxframe/dataframe/sort/sort_index.py,sha256=Hwbkm9x8kqGgXh4gMcAtYMDjYtt-S3CJXfYR9pN5Iqk,5412
|
|
94
96
|
maxframe/dataframe/sort/__init__.py,sha256=Vt2Ynr7uAX51hLbQu93QeHiFH4D9_WJMO99KljpbO2U,1160
|
|
@@ -126,9 +128,9 @@ maxframe/dataframe/extensions/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0
|
|
|
126
128
|
maxframe/dataframe/groupby/aggregation.py,sha256=BuhqZal6RLHkjvwJep86oT1a7rMqxxUAPxQ_dao6I6E,11953
|
|
127
129
|
maxframe/dataframe/groupby/fill.py,sha256=JF3NxyXigZqg8ecKtLSndDmNMX8S6g_ChQR9JAK036E,4721
|
|
128
130
|
maxframe/dataframe/groupby/cum.py,sha256=jdGQm7U-GosgHMfneHZC5Z2uraj6iBmSFOhqP3m18B0,3755
|
|
129
|
-
maxframe/dataframe/groupby/__init__.py,sha256=
|
|
131
|
+
maxframe/dataframe/groupby/__init__.py,sha256=nZkz1OAdYRj8qwQkUAZDax0pfCsUH_fprwuksS97vuc,3361
|
|
130
132
|
maxframe/dataframe/groupby/getitem.py,sha256=kUcI9oIrjOcAHnho96Le9yEJxFydALsWbGpZfTtF8gY,3252
|
|
131
|
-
maxframe/dataframe/groupby/core.py,sha256=
|
|
133
|
+
maxframe/dataframe/groupby/core.py,sha256=K1hg9jod6z3C65SYoidmEAd_k0Mear4l5IQuwNMjpxQ,6075
|
|
132
134
|
maxframe/dataframe/groupby/transform.py,sha256=pY3WPA4gN8piYSTInncjnRdh8mi9FDQa00A-Pyaoil4,8586
|
|
133
135
|
maxframe/dataframe/groupby/head.py,sha256=ZDkbSn3HuUR4GGkZJqo_fL-6KFJfs55aKXQkAh_0wvA,3266
|
|
134
136
|
maxframe/dataframe/groupby/sample.py,sha256=IdoyzT-V5309txYvM_iaYKupsULfozMGwm1K3oihTf4,6935
|
|
@@ -171,8 +173,8 @@ maxframe/dataframe/plotting/__init__.py,sha256=JShRu5lSUAfwgTFUMxQoIqiH1gg9H5cie
|
|
|
171
173
|
maxframe/dataframe/plotting/core.py,sha256=-Ei1ZMQbaTSC6tk8olHmoLuxL7C7NbhKQ6gnSpBuHFw,2193
|
|
172
174
|
maxframe/dataframe/plotting/tests/test_plotting.py,sha256=IKAOcG0LotMGZCrvDzWiHmoFXqtyrVDn87eIOdyg6Ho,4118
|
|
173
175
|
maxframe/dataframe/plotting/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
|
|
174
|
-
maxframe/dataframe/window/aggregation.py,sha256=
|
|
175
|
-
maxframe/dataframe/window/ewm.py,sha256=
|
|
176
|
+
maxframe/dataframe/window/aggregation.py,sha256=ncdJavwZcj6LWfF_W9wWUWAIjOxu7DOEEW16y5EQCF4,3794
|
|
177
|
+
maxframe/dataframe/window/ewm.py,sha256=xqRiw4_KYpPTzhdKnN9wIknWdKsFls-uEzYO1T18vDk,7789
|
|
176
178
|
maxframe/dataframe/window/__init__.py,sha256=D8DOhmH-PabNAFPNwzwsw-mH0iELIwyU7tyiiaZGJhw,910
|
|
177
179
|
maxframe/dataframe/window/core.py,sha256=T15k5OqC1y1ewlT2ddDf0oFP9LeOdneZ3po1mxLdOgc,2205
|
|
178
180
|
maxframe/dataframe/window/rolling.py,sha256=btt0uOtsK3z2gR96tVXrXJUrvLFus5FUunRaooizpIw,12042
|
|
@@ -180,7 +182,7 @@ maxframe/dataframe/window/expanding.py,sha256=OmrNK14cj9corIyTCACZy1_SgTXL3i2ZQp
|
|
|
180
182
|
maxframe/dataframe/window/tests/test_expanding.py,sha256=RV982SmVAVuUVS7cjWdTFk_VqACphhRQb9DPu9eyWqQ,1944
|
|
181
183
|
maxframe/dataframe/window/tests/test_rolling.py,sha256=JWnXGUCz3li5B10rvnzzcTXbQq1lL6Svo3NL7BeBGGU,1714
|
|
182
184
|
maxframe/dataframe/window/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
|
|
183
|
-
maxframe/dataframe/window/tests/test_ewm.py,sha256=
|
|
185
|
+
maxframe/dataframe/window/tests/test_ewm.py,sha256=R6I2smeLhpS9cczCAuZdP5nKj0s4ssKB5VJ6hRW-9b0,2060
|
|
184
186
|
maxframe/dataframe/arithmetic/arctan.py,sha256=zFCdG8rCmRmU3p9jGm3tQp5ulAsx4nmKn3m8e7jY6nk,930
|
|
185
187
|
maxframe/dataframe/arithmetic/arccos.py,sha256=WF3SE975VbwCOsrSPNck4swIejDFI-tVm9H4DQrBv6Q,930
|
|
186
188
|
maxframe/dataframe/arithmetic/arctanh.py,sha256=7H32t46reUr0k_4pEj2Dx_dAkH59OIUinedqoOPd0Hg,935
|
|
@@ -265,15 +267,15 @@ maxframe/core/entity/tileables.py,sha256=b9jn_OQ-FQkbw7E7jMLjoJ4-VR7tBS8Mbx_j4iZ
|
|
|
265
267
|
maxframe/core/entity/__init__.py,sha256=Hz_p6eTkrSdkT7YCo5aeGZ33tms5wwifMp4TeYsAVlw,1292
|
|
266
268
|
maxframe/core/entity/core.py,sha256=t7Ex9Yb7A1h_XwyRG88Fx4ZOai-NQKi2luRVS_jFPEo,4018
|
|
267
269
|
maxframe/core/entity/utils.py,sha256=IuNgFmBQFRioAA1hgZe6nTEggOmDY-iooZqncQQrV28,942
|
|
268
|
-
maxframe/core/entity/executable.py,sha256=
|
|
270
|
+
maxframe/core/entity/executable.py,sha256=HKXHXdPIyxg9i-OWmJxIY3KfXwX0x3xN9QcR5Xhc7dQ,10938
|
|
269
271
|
maxframe/core/entity/output_types.py,sha256=uqApvFK8w6_aMxRets69dTwD1ndBDgVgqDCflyt9ubg,2645
|
|
270
|
-
maxframe/core/entity/objects.py,sha256=
|
|
272
|
+
maxframe/core/entity/objects.py,sha256=RMHLTGbIHZNxxX59lAuQydAKcR32qKleIYUqdElGS4E,3034
|
|
271
273
|
maxframe/core/entity/fuse.py,sha256=47U6MHRzA2ZvUi-kJb7b3mC_gN07x3yebBgX2Jj7VZo,2277
|
|
272
274
|
maxframe/core/entity/chunks.py,sha256=yNSLCWOpA_Z6aGr6ZI32dIJf3xPdRBWbvdsl8sTM3BE,2134
|
|
273
275
|
maxframe/core/graph/__init__.py,sha256=rnsXwW0ouh1f7SVtq73-PzLE-MBM6Op_0l6J7b7wGRE,821
|
|
274
276
|
maxframe/core/graph/entity.py,sha256=56gjXyDXN-TTPm3AQOxuRVQbb_fguKFDL_Xm7i95XEk,5559
|
|
275
|
-
maxframe/core/graph/core.cpython-38-darwin.so,sha256=
|
|
276
|
-
maxframe/core/graph/core.pyx,sha256=
|
|
277
|
+
maxframe/core/graph/core.cpython-38-darwin.so,sha256=bzoZXzNoXK_4k42vsmY7HDkcXsRVwqZ_MsJii_QtHyE,731606
|
|
278
|
+
maxframe/core/graph/core.pyx,sha256=ZJPx_MTOBMaX-6mns6tAiu-wrIBvRAKN44YAGTypJ1Y,15887
|
|
277
279
|
maxframe/core/graph/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
|
|
278
280
|
maxframe/core/graph/tests/test_graph.py,sha256=kZfe_SfMOoHjEfXvtVtn0RjK4lOd-tFasxSpfL4G1WE,7462
|
|
279
281
|
maxframe/core/graph/builder/__init__.py,sha256=BV5tAiZ-bJE0FBGSCEnfqKNwBNHgvB9aSzEp0PY6SnA,702
|
|
@@ -289,16 +291,16 @@ maxframe/core/operator/core.py,sha256=T2YYIleKiFxmufpHF7ELOztlJsYO-Kn4KYhQhY4jRU
|
|
|
289
291
|
maxframe/core/operator/shuffle.py,sha256=A_w62UxXRKyDNESo4Q0UGmCpuFfPlQXWmPyQGHdpz3A,4746
|
|
290
292
|
maxframe/core/operator/objects.py,sha256=SJoF5GBusb1_j3-dSnfKSndr9WTrlU3Db-fCrcMs3jI,2152
|
|
291
293
|
maxframe/core/operator/fuse.py,sha256=0RGemF99gQCwV4aEk-K6T5KAGToO-487dFk8LyYDIZw,925
|
|
292
|
-
maxframe/core/operator/base.py,sha256=
|
|
294
|
+
maxframe/core/operator/base.py,sha256=nxuSKjbBzDrItM9PGmFo8RLwParazu525jMLWj0kXkM,15251
|
|
293
295
|
maxframe/core/operator/tests/test_core.py,sha256=57aICnc5VLqdVK7icAORTWC81bSjBxeeVWIJcha9J_0,1691
|
|
294
296
|
maxframe/core/operator/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
|
|
295
|
-
maxframe/config/config.py,sha256=
|
|
297
|
+
maxframe/config/config.py,sha256=3lQj99eMGg9MBW1gMJAdGFD88UEcdZf71sHgAHXASAk,13045
|
|
296
298
|
maxframe/config/validators.py,sha256=2m9MrkjDUFiU4PPaWIw8tjwMaOy8AYmuJFqVnnY8IMY,1615
|
|
297
299
|
maxframe/config/__init__.py,sha256=g5lN3nP2HTAXa6ExGxU1NwU1M9ulYPmAcsV-gU7nIW8,656
|
|
298
300
|
maxframe/config/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
|
|
299
301
|
maxframe/config/tests/test_validators.py,sha256=U_7yKSl0FdVdDwKU1EsnCzNWaOXi8xrIC08x7hb_O4c,1240
|
|
300
302
|
maxframe/config/tests/test_config.py,sha256=Cx3buKli4F77zwJaB-vnUSlE9LUbt3ObHW38cIE2dDs,2736
|
|
301
|
-
maxframe/serialization/exception.py,sha256=
|
|
303
|
+
maxframe/serialization/exception.py,sha256=TIlEDiuYwVgYkN4Se0ydXn2KzEJy4Yv8w2XwlTBqDdM,3027
|
|
302
304
|
maxframe/serialization/core.pxd,sha256=51XCBozW9j_E0sBgyYkHf5xNfqCMJb36Fwm_Pxv_rZs,1303
|
|
303
305
|
maxframe/serialization/pandas.py,sha256=D4_H4KjAl8DRQtODh9VBP94yB8ce1cYQUKrzXtn10KE,7147
|
|
304
306
|
maxframe/serialization/arrow.py,sha256=VnGxNLU9UV_cUPTze43bEFCIbYLAOZnp2pAwVJbAIzQ,3418
|
|
@@ -306,13 +308,13 @@ maxframe/serialization/__init__.py,sha256=9eSnoDww1uw2DAXEBBTB2atJQHzd-38XVxrCko
|
|
|
306
308
|
maxframe/serialization/maxframe_objects.py,sha256=R9WEjbHL0Kr56OGkYDU9fcGi7gII6fGlXhi6IyihTsM,1365
|
|
307
309
|
maxframe/serialization/numpy.py,sha256=8_GSo45l_eNoMn4NAGEb9NLXY_9i4tf9KK4EzG0mKpA,3213
|
|
308
310
|
maxframe/serialization/scipy.py,sha256=hP0fAW0di9UgJrGtANB2S8hLDbFBtR8p5NDqAMt5rDI,2427
|
|
309
|
-
maxframe/serialization/core.cpython-38-darwin.so,sha256=
|
|
311
|
+
maxframe/serialization/core.cpython-38-darwin.so,sha256=_PvDZERhV7sTftKQCEPcJGCAgxlOHLeKu6GvPFGms8c,1308518
|
|
310
312
|
maxframe/serialization/core.pyx,sha256=AATN47RdBTq2zg7--3xX2VHyAZSvoAuYRt7B7gEgKPE,33984
|
|
311
313
|
maxframe/serialization/tests/test_serial.py,sha256=Wj_I6CBQMaOtE8WtqdUaBoU8FhBOihht6SfeHOJV-zU,12511
|
|
312
314
|
maxframe/serialization/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
|
|
313
|
-
maxframe/serialization/serializables/field.py,sha256=
|
|
315
|
+
maxframe/serialization/serializables/field.py,sha256=atVgX-9rsVG1fTev7vjQArVwIEaCRjXoSEjpQ3mh6bA,16015
|
|
314
316
|
maxframe/serialization/serializables/__init__.py,sha256=_wyFZF5QzSP32wSXlXHEPl98DN658I66WamP8XPJy0c,1351
|
|
315
|
-
maxframe/serialization/serializables/core.py,sha256=
|
|
317
|
+
maxframe/serialization/serializables/core.py,sha256=xlqVUlBK3aLTavHLWHg4JXUTaBGzSuM7t-XHahB8et4,8965
|
|
316
318
|
maxframe/serialization/serializables/field_type.py,sha256=Feh09hu8XyaxS5MaJ4za_pcvqJVuMkOeGxwQ9OuJw6I,14865
|
|
317
319
|
maxframe/serialization/serializables/tests/test_field_type.py,sha256=T3ebXbUkKveC9Pq1nIl85e4eYascFeJ52d0REHbz5jo,4381
|
|
318
320
|
maxframe/serialization/serializables/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
|
|
@@ -328,14 +330,15 @@ maxframe/odpsio/tests/test_schema.py,sha256=yss1ly55ErYse95XMFq2s_GWL8UnwZga5RyN
|
|
|
328
330
|
maxframe/odpsio/tests/test_arrow.py,sha256=SQ9EmI9_VOOC8u6Rg6nh3IPC2fPbLvJ9HwtpMNDRhL8,3106
|
|
329
331
|
maxframe/odpsio/tests/test_volumeio.py,sha256=UEqFANuPKyFtlIh2JNi-LoixH52bxsgHdxu3himnEvs,3022
|
|
330
332
|
maxframe/tests/test_utils.py,sha256=xaAoURr5NOJUTY0XVa2H8qOStcEH5UQSXItkatHFxFE,11977
|
|
331
|
-
maxframe/tests/test_protocol.py,sha256=
|
|
333
|
+
maxframe/tests/test_protocol.py,sha256=t11yxh4_gWxxCuk09zo3pn9Nn96DBBQTBt12ewKDwLQ,5187
|
|
332
334
|
maxframe/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
|
|
333
|
-
maxframe/tests/utils.py,sha256=
|
|
335
|
+
maxframe/tests/utils.py,sha256=wJtSFXt3BD4i5zdO4JBQk_kNAxrtyGLro0jodCA4xuY,4568
|
|
336
|
+
maxframe/tests/test_codegen.py,sha256=GMrnpSb2eyB_nmuv8-_p47Kw877ElKS3BP52SpqZNIQ,2208
|
|
334
337
|
maxframe/lib/wrapped_pickle.py,sha256=xJa0wI-GsBZFKQpVnlh_hZBlQ2u1D8VO2aBIW7VOdP4,3810
|
|
335
338
|
maxframe/lib/version.py,sha256=yQ6HkDOvU9X1rpI49auh-qku2g7gIiztgEH6v1urOrk,18321
|
|
336
339
|
maxframe/lib/compression.py,sha256=k9DSrl_dNBsn5azLjBdL5B4WZ6eNvmCrdMbcF1G7JSc,1442
|
|
337
340
|
maxframe/lib/__init__.py,sha256=CzfbLNqqm1yR1i6fDwCd4h1ptuKVDbURFVCb0ra7QNc,642
|
|
338
|
-
maxframe/lib/mmh3.cpython-38-darwin.so,sha256=
|
|
341
|
+
maxframe/lib/mmh3.cpython-38-darwin.so,sha256=9tUwuoZ0r-_FvVt2lsA2JP1Y2EH7YvDwfiHWFZE3P-I,120374
|
|
339
342
|
maxframe/lib/functools_compat.py,sha256=PMSkct9GIbzq-aBwTnggrOLNfLh4xQnYTIFMPblzCUA,2616
|
|
340
343
|
maxframe/lib/mmh3_src/mmh3module.cpp,sha256=9J9eA42eKWTl546fvfQPNuIM3B2jpWSADpgIw3tr2jg,11604
|
|
341
344
|
maxframe/lib/mmh3_src/MurmurHash3.h,sha256=lg5uXUFyMBge2BWRn0FgrqaCFCMfDWoTXD4PQtjHrMA,1263
|
|
@@ -344,7 +347,7 @@ maxframe/lib/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZH
|
|
|
344
347
|
maxframe/lib/tests/test_wrapped_pickle.py,sha256=oz1RLwHSZstXgw4caNeaD0ZgQZvkzDLsx7hFN-NvP7U,1524
|
|
345
348
|
maxframe/lib/cython/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
|
|
346
349
|
maxframe/lib/cython/libcpp.pxd,sha256=2o9HWtyCMtN9xk7WH_mq1i89IbMPwfZA8yHETjRALXs,1100
|
|
347
|
-
maxframe/lib/aio/isolation.py,sha256=
|
|
350
|
+
maxframe/lib/aio/isolation.py,sha256=2nA16GdOXEUNVVdxQXbmU4YvY6wPG2oSV4z1x9uW6Gw,2761
|
|
348
351
|
maxframe/lib/aio/__init__.py,sha256=1_nx7d5AqXRwa7ODzVfL8gH-tsDAH4YyY-JFPC8TA6w,936
|
|
349
352
|
maxframe/lib/aio/_threads.py,sha256=tr7FYViGT7nyR7HRw3vE3W-9r3-dZ1IP_Kbhe9sgqpw,1328
|
|
350
353
|
maxframe/lib/aio/file.py,sha256=aZF8NkkccsVsOniWMBiPqPSk48bb7zGRPB2BegWRaqM,2012
|
|
@@ -384,7 +387,7 @@ maxframe/lib/tblib/cpython.py,sha256=FQ0f6WTQyQHoMRhgPqrA0y0Ygxlbj5IC53guxA4h9Cw
|
|
|
384
387
|
maxframe/lib/tblib/decorators.py,sha256=bcllK3kVuPnj6SNZGmlJGxTK0ovdt7TJDXrhA4UE5sQ,1063
|
|
385
388
|
maxframe/tensor/array_utils.py,sha256=259vG4SjyhiheARCZeEnfJdZjoojyrELn41oRcyAELs,4943
|
|
386
389
|
maxframe/tensor/__init__.py,sha256=-kir8LUsXCDGcc7YdKqWgNEHSrgU_HE5uPam0jLLP6g,3511
|
|
387
|
-
maxframe/tensor/core.py,sha256=
|
|
390
|
+
maxframe/tensor/core.py,sha256=Ojxaf5b8sJ6ZZGezyFHQJ5XsSpUrBOnZgFeUQgpVJpI,21914
|
|
388
391
|
maxframe/tensor/utils.py,sha256=bwVN0iuVic1tpFai6Hk-1tQLqckQ2IYS7yZKMTcOU1I,22914
|
|
389
392
|
maxframe/tensor/operators.py,sha256=iGkDIRz152gXrPb5JbqOvXngpq3QaCg-aNO4gHZPLN0,3461
|
|
390
393
|
maxframe/tensor/statistics/quantile.py,sha256=UFzTmBwgNL7k_QOJ84qPfycQrW8MyOa1gcp-uFsylIY,9484
|
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
|
|
15
15
|
from typing import Any, Dict, List
|
|
16
16
|
|
|
17
|
+
import msgpack
|
|
17
18
|
from tornado import httpclient
|
|
18
19
|
|
|
19
20
|
from maxframe.core import TileableGraph
|
|
@@ -28,7 +29,6 @@ from maxframe.protocol import (
|
|
|
28
29
|
)
|
|
29
30
|
from maxframe.typing_ import TimeoutType
|
|
30
31
|
from maxframe.utils import (
|
|
31
|
-
deserialize_serializable,
|
|
32
32
|
format_timeout_params,
|
|
33
33
|
serialize_serializable,
|
|
34
34
|
wait_http_response,
|
|
@@ -47,12 +47,12 @@ class FrameDriverClient:
|
|
|
47
47
|
resp = await httpclient.AsyncHTTPClient().fetch(
|
|
48
48
|
req_url, method="POST", body=serialize_serializable(req_body)
|
|
49
49
|
)
|
|
50
|
-
return
|
|
50
|
+
return SessionInfo.from_json(msgpack.loads(resp.body))
|
|
51
51
|
|
|
52
52
|
async def get_session(self, session_id: str) -> SessionInfo:
|
|
53
53
|
req_url = f"{self._endpoint}/api/sessions/{session_id}"
|
|
54
54
|
resp = await httpclient.AsyncHTTPClient().fetch(req_url, method="GET")
|
|
55
|
-
return
|
|
55
|
+
return SessionInfo.from_json(msgpack.loads(resp.body))
|
|
56
56
|
|
|
57
57
|
async def delete_session(self, session_id: str):
|
|
58
58
|
req_url = f"{self._endpoint}/api/sessions/{session_id}"
|
|
@@ -71,12 +71,12 @@ class FrameDriverClient:
|
|
|
71
71
|
method="POST",
|
|
72
72
|
body=serialize_serializable(ProtocolBody(body=req_body)),
|
|
73
73
|
)
|
|
74
|
-
return
|
|
74
|
+
return DagInfo.from_json(msgpack.loads(resp.body))
|
|
75
75
|
|
|
76
76
|
async def get_dag_info(self, session_id: str, dag_id: str) -> DagInfo:
|
|
77
77
|
req_url = f"{self._endpoint}/api/sessions/{session_id}/dags/{dag_id}"
|
|
78
78
|
resp = await httpclient.AsyncHTTPClient().fetch(req_url, method="GET")
|
|
79
|
-
return
|
|
79
|
+
return DagInfo.from_json(msgpack.loads(resp.body))
|
|
80
80
|
|
|
81
81
|
async def wait_dag(self, session_id: str, dag_id: str, timeout: TimeoutType = None):
|
|
82
82
|
query_part = format_timeout_params(timeout)
|
|
@@ -87,7 +87,7 @@ class FrameDriverClient:
|
|
|
87
87
|
resp = await wait_http_response(
|
|
88
88
|
req_url, method="GET", request_timeout=timeout
|
|
89
89
|
)
|
|
90
|
-
info =
|
|
90
|
+
info = DagInfo.from_json(msgpack.loads(resp.body))
|
|
91
91
|
except TimeoutError:
|
|
92
92
|
info = await self.get_dag_info(session_id, dag_id)
|
|
93
93
|
return info
|
|
@@ -103,7 +103,7 @@ class FrameDriverClient:
|
|
|
103
103
|
resp = await wait_http_response(
|
|
104
104
|
req_url, method="DELETE", request_timeout=timeout
|
|
105
105
|
)
|
|
106
|
-
info =
|
|
106
|
+
info = DagInfo.from_json(msgpack.loads(resp.body))
|
|
107
107
|
except TimeoutError:
|
|
108
108
|
info = await self.get_dag_info(session_id, dag_id)
|
|
109
109
|
return info
|
maxframe_client/session/task.py
CHANGED
|
@@ -22,6 +22,7 @@ import msgpack
|
|
|
22
22
|
from odps import ODPS
|
|
23
23
|
from odps import options as odps_options
|
|
24
24
|
from odps import serializers
|
|
25
|
+
from odps.errors import parse_instance_error
|
|
25
26
|
from odps.models import Instance, Task
|
|
26
27
|
|
|
27
28
|
from maxframe.config import options
|
|
@@ -29,6 +30,11 @@ from maxframe.core import TileableGraph
|
|
|
29
30
|
from maxframe.protocol import DagInfo, JsonSerializable, ResultInfo, SessionInfo
|
|
30
31
|
from maxframe.utils import deserialize_serializable, serialize_serializable, to_str
|
|
31
32
|
|
|
33
|
+
try:
|
|
34
|
+
from maxframe import __version__ as mf_version
|
|
35
|
+
except ImportError:
|
|
36
|
+
mf_version = None
|
|
37
|
+
|
|
32
38
|
from .consts import (
|
|
33
39
|
MAXFRAME_DEFAULT_PROTOCOL,
|
|
34
40
|
MAXFRAME_OUTPUT_JSON_FORMAT,
|
|
@@ -91,6 +97,8 @@ class MaxFrameTask(Task):
|
|
|
91
97
|
"odps.maxframe.output_format": self._output_format,
|
|
92
98
|
"odps.service.endpoint": self._service_endpoint,
|
|
93
99
|
}
|
|
100
|
+
if mf_version:
|
|
101
|
+
mf_opts["odps.maxframe.client_version"] = mf_version
|
|
94
102
|
settings.update(mf_opts)
|
|
95
103
|
self.properties["settings"] = json.dumps(settings)
|
|
96
104
|
return super().serial()
|
|
@@ -104,13 +112,19 @@ class MaxFrameInstanceCaller(MaxFrameServiceCaller):
|
|
|
104
112
|
odps_entry: ODPS,
|
|
105
113
|
task_name: Optional[str] = None,
|
|
106
114
|
project: Optional[str] = None,
|
|
107
|
-
priority: Optional[
|
|
115
|
+
priority: Optional[int] = None,
|
|
108
116
|
running_cluster: Optional[str] = None,
|
|
109
117
|
nested_instance_id: Optional[str] = None,
|
|
110
118
|
major_version: Optional[str] = None,
|
|
111
119
|
output_format: Optional[str] = None,
|
|
112
120
|
**kwargs,
|
|
113
121
|
):
|
|
122
|
+
if callable(odps_options.get_priority):
|
|
123
|
+
default_priority = odps_options.get_priority(odps_entry)
|
|
124
|
+
else:
|
|
125
|
+
default_priority = odps_options.priority
|
|
126
|
+
priority = priority if priority is not None else default_priority
|
|
127
|
+
|
|
114
128
|
self._odps_entry = odps_entry
|
|
115
129
|
self._task_name = task_name
|
|
116
130
|
self._project = project
|
|
@@ -118,6 +132,7 @@ class MaxFrameInstanceCaller(MaxFrameServiceCaller):
|
|
|
118
132
|
self._running_cluster = running_cluster
|
|
119
133
|
self._major_version = major_version
|
|
120
134
|
self._output_format = output_format or MAXFRAME_OUTPUT_MSGPACK_FORMAT
|
|
135
|
+
|
|
121
136
|
if nested_instance_id is None:
|
|
122
137
|
self._nested = False
|
|
123
138
|
self._instance = None
|
|
@@ -126,9 +141,12 @@ class MaxFrameInstanceCaller(MaxFrameServiceCaller):
|
|
|
126
141
|
self._instance = odps_entry.get_instance(nested_instance_id)
|
|
127
142
|
|
|
128
143
|
def _deserial_task_info_result(
|
|
129
|
-
self, content: Union[bytes, str], target_cls: Type[JsonSerializable]
|
|
144
|
+
self, content: Union[bytes, str, dict], target_cls: Type[JsonSerializable]
|
|
130
145
|
):
|
|
131
|
-
|
|
146
|
+
if isinstance(content, (str, bytes)):
|
|
147
|
+
json_data = json.loads(to_str(content))
|
|
148
|
+
else:
|
|
149
|
+
json_data = content
|
|
132
150
|
result_data = base64.b64decode(json_data["result"])
|
|
133
151
|
if self._output_format == MAXFRAME_OUTPUT_MAXFRAME_FORMAT:
|
|
134
152
|
return deserialize_serializable(result_data)
|
|
@@ -166,12 +184,22 @@ class MaxFrameInstanceCaller(MaxFrameServiceCaller):
|
|
|
166
184
|
)
|
|
167
185
|
return self._deserial_task_info_result(result, SessionInfo)
|
|
168
186
|
|
|
187
|
+
def _parse_instance_result_error(self):
|
|
188
|
+
result_data = self._instance.get_task_result(self._task_name)
|
|
189
|
+
try:
|
|
190
|
+
info = self._deserial_task_info_result({"result": result_data}, SessionInfo)
|
|
191
|
+
except:
|
|
192
|
+
raise parse_instance_error(result_data)
|
|
193
|
+
info.error_info.reraise()
|
|
194
|
+
|
|
169
195
|
def _wait_instance_task_ready(
|
|
170
196
|
self, interval: float = 0.1, max_interval: float = 5.0, timeout: int = None
|
|
171
197
|
):
|
|
172
198
|
check_time = time.time()
|
|
173
199
|
timeout = timeout or options.client.task_start_timeout
|
|
174
200
|
while True:
|
|
201
|
+
if self._instance.is_terminated(retry=True):
|
|
202
|
+
self._parse_instance_result_error()
|
|
175
203
|
status_json = json.loads(
|
|
176
204
|
self._instance.get_task_info(self._task_name, "status") or "{}"
|
|
177
205
|
)
|
|
@@ -18,6 +18,7 @@ import os
|
|
|
18
18
|
import mock
|
|
19
19
|
from defusedxml import ElementTree
|
|
20
20
|
from odps import ODPS
|
|
21
|
+
from odps import options as odps_options
|
|
21
22
|
|
|
22
23
|
from ...session.consts import MAXFRAME_OUTPUT_JSON_FORMAT
|
|
23
24
|
from ...session.task import MaxFrameInstanceCaller, MaxFrameTask
|
|
@@ -27,17 +28,20 @@ expected_file_dir = os.path.join(os.path.dirname(__file__), "expected-data")
|
|
|
27
28
|
|
|
28
29
|
def test_maxframe_instance_caller_creating_session():
|
|
29
30
|
o = ODPS.from_environments()
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
31
|
+
|
|
32
|
+
def create_caller(**kwargs):
|
|
33
|
+
kw = dict(
|
|
34
|
+
odps_entry=o,
|
|
35
|
+
task_name="task_test",
|
|
36
|
+
major_version="test_version",
|
|
37
|
+
output_format=MAXFRAME_OUTPUT_JSON_FORMAT,
|
|
38
|
+
running_cluster="test_cluster",
|
|
39
|
+
)
|
|
40
|
+
kw.update(**kwargs)
|
|
41
|
+
return MaxFrameInstanceCaller(**kw)
|
|
38
42
|
|
|
39
43
|
def mock_create(self, task: MaxFrameTask, priority=None, running_cluster=None):
|
|
40
|
-
assert priority ==
|
|
44
|
+
assert priority == 100
|
|
41
45
|
assert running_cluster == "test_cluster"
|
|
42
46
|
root = ElementTree.parse(
|
|
43
47
|
os.path.join(expected_file_dir, "create_session.xml")
|
|
@@ -62,6 +66,20 @@ def test_maxframe_instance_caller_creating_session():
|
|
|
62
66
|
target="maxframe_client.session.task.MaxFrameInstanceCaller",
|
|
63
67
|
_wait_instance_task_ready=mock.DEFAULT,
|
|
64
68
|
get_session=mock.DEFAULT,
|
|
65
|
-
):
|
|
66
|
-
|
|
69
|
+
), mock.patch("odps.models.instances.BaseInstances.create", mock_create):
|
|
70
|
+
task_caller = create_caller(priority=100)
|
|
71
|
+
task_caller.create_session()
|
|
72
|
+
|
|
73
|
+
old_priority = odps_options.priority
|
|
74
|
+
old_get_priority = odps_options.get_priority
|
|
75
|
+
try:
|
|
76
|
+
task_caller = create_caller(priority=100)
|
|
77
|
+
odps_options.priority = 100
|
|
78
|
+
task_caller.create_session()
|
|
79
|
+
|
|
80
|
+
odps_options.priority = None
|
|
81
|
+
odps_options.get_priority = lambda _: 100
|
|
67
82
|
task_caller.create_session()
|
|
83
|
+
finally:
|
|
84
|
+
odps_options.priority = old_priority
|
|
85
|
+
odps_options.get_priority = old_get_priority
|
|
@@ -24,6 +24,7 @@ from odps import ODPS
|
|
|
24
24
|
import maxframe.dataframe as md
|
|
25
25
|
import maxframe.remote as mr
|
|
26
26
|
from maxframe.core import ExecutableTuple, TileableGraph
|
|
27
|
+
from maxframe.lib.aio import stop_isolation
|
|
27
28
|
from maxframe.protocol import ResultInfo
|
|
28
29
|
from maxframe.serialization import RemoteException
|
|
29
30
|
from maxframe.session import new_session
|
|
@@ -52,6 +53,7 @@ def start_mock_session(framedriver_app): # noqa: F811
|
|
|
52
53
|
time.sleep(5) # Wait for temp table deleted
|
|
53
54
|
else:
|
|
54
55
|
session.reset_default()
|
|
56
|
+
stop_isolation()
|
|
55
57
|
|
|
56
58
|
|
|
57
59
|
def test_simple_run_dataframe(start_mock_session):
|
|
File without changes
|