maxframe 0.1.0b4__cp310-cp310-macosx_10_9_universal2.whl → 0.1.0b5__cp310-cp310-macosx_10_9_universal2.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of maxframe might be problematic. Click here for more details.
- maxframe/__init__.py +1 -0
- maxframe/_utils.cpython-310-darwin.so +0 -0
- maxframe/codegen.py +46 -1
- maxframe/config/config.py +11 -1
- maxframe/core/graph/core.cpython-310-darwin.so +0 -0
- maxframe/dataframe/__init__.py +1 -0
- maxframe/dataframe/core.py +30 -8
- maxframe/dataframe/datasource/read_odps_query.py +3 -1
- maxframe/dataframe/datasource/read_odps_table.py +3 -1
- maxframe/dataframe/misc/__init__.py +4 -0
- maxframe/dataframe/misc/apply.py +1 -1
- maxframe/dataframe/misc/case_when.py +141 -0
- maxframe/dataframe/misc/pivot_table.py +262 -0
- maxframe/dataframe/misc/tests/test_misc.py +61 -0
- maxframe/dataframe/plotting/core.py +2 -2
- maxframe/dataframe/reduction/core.py +2 -1
- maxframe/dataframe/utils.py +7 -0
- maxframe/learn/contrib/utils.py +52 -0
- maxframe/learn/contrib/xgboost/__init__.py +26 -0
- maxframe/learn/contrib/xgboost/classifier.py +86 -0
- maxframe/learn/contrib/xgboost/core.py +156 -0
- maxframe/learn/contrib/xgboost/dmatrix.py +150 -0
- maxframe/learn/contrib/xgboost/predict.py +138 -0
- maxframe/learn/contrib/xgboost/regressor.py +78 -0
- maxframe/learn/contrib/xgboost/tests/__init__.py +13 -0
- maxframe/learn/contrib/xgboost/tests/test_core.py +43 -0
- maxframe/learn/contrib/xgboost/train.py +121 -0
- maxframe/learn/utils/__init__.py +15 -0
- maxframe/learn/utils/core.py +29 -0
- maxframe/lib/mmh3.cpython-310-darwin.so +0 -0
- maxframe/odpsio/arrow.py +2 -3
- maxframe/odpsio/tableio.py +22 -0
- maxframe/odpsio/tests/test_schema.py +16 -11
- maxframe/opcodes.py +3 -0
- maxframe/serialization/core.cpython-310-darwin.so +0 -0
- maxframe/serialization/core.pyi +61 -0
- maxframe/session.py +28 -0
- maxframe/tensor/__init__.py +1 -1
- maxframe/tensor/base/__init__.py +2 -0
- maxframe/tensor/base/atleast_1d.py +74 -0
- maxframe/tensor/base/unique.py +205 -0
- maxframe/tensor/datasource/array.py +4 -2
- maxframe/tensor/datasource/scalar.py +1 -1
- maxframe/udf.py +63 -3
- maxframe/utils.py +6 -0
- {maxframe-0.1.0b4.dist-info → maxframe-0.1.0b5.dist-info}/METADATA +2 -2
- {maxframe-0.1.0b4.dist-info → maxframe-0.1.0b5.dist-info}/RECORD +53 -36
- maxframe_client/fetcher.py +65 -3
- maxframe_client/session/odps.py +30 -1
- maxframe_client/session/task.py +26 -53
- maxframe_client/tests/test_session.py +28 -1
- {maxframe-0.1.0b4.dist-info → maxframe-0.1.0b5.dist-info}/WHEEL +0 -0
- {maxframe-0.1.0b4.dist-info → maxframe-0.1.0b5.dist-info}/top_level.txt +0 -0
maxframe/udf.py
CHANGED
|
@@ -12,21 +12,51 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
+
import shlex
|
|
15
16
|
from typing import Callable, List, Optional, Union
|
|
16
17
|
|
|
17
18
|
from odps.models import Resource
|
|
18
19
|
|
|
19
20
|
from .serialization.serializables import (
|
|
21
|
+
BoolField,
|
|
20
22
|
FieldTypes,
|
|
21
23
|
FunctionField,
|
|
22
24
|
ListField,
|
|
23
25
|
Serializable,
|
|
26
|
+
StringField,
|
|
24
27
|
)
|
|
28
|
+
from .utils import tokenize
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class PythonPackOptions(Serializable):
|
|
32
|
+
key = StringField("key")
|
|
33
|
+
requirements = ListField("requirements", FieldTypes.string, default_factory=list)
|
|
34
|
+
force_rebuild = BoolField("force_rebuild", default=False)
|
|
35
|
+
prefer_binary = BoolField("prefer_binary", default=False)
|
|
36
|
+
pre_release = BoolField("pre_release", default=False)
|
|
37
|
+
pack_instance_id = StringField("pack_instance_id", default=None)
|
|
38
|
+
|
|
39
|
+
def __init__(self, key: str = None, **kw):
|
|
40
|
+
super().__init__(key=key, **kw)
|
|
41
|
+
if self.key is None:
|
|
42
|
+
args = {
|
|
43
|
+
"force_rebuild": self.force_rebuild,
|
|
44
|
+
"prefer_binary": self.prefer_binary,
|
|
45
|
+
"pre_release": self.pre_release,
|
|
46
|
+
}
|
|
47
|
+
self.key = tokenize(set(self.requirements), args)
|
|
48
|
+
|
|
49
|
+
def __repr__(self):
|
|
50
|
+
return (
|
|
51
|
+
f"<PythonPackOptions {self.requirements} force_rebuild={self.force_rebuild} "
|
|
52
|
+
f"prefer_binary={self.prefer_binary} pre_release={self.pre_release}>"
|
|
53
|
+
)
|
|
25
54
|
|
|
26
55
|
|
|
27
56
|
class MarkedFunction(Serializable):
|
|
28
57
|
func = FunctionField("func")
|
|
29
58
|
resources = ListField("resources", FieldTypes.string, default_factory=list)
|
|
59
|
+
pythonpacks = ListField("pythonpacks", FieldTypes.reference, default_factory=list)
|
|
30
60
|
|
|
31
61
|
def __init__(self, func: Optional[Callable] = None, **kw):
|
|
32
62
|
super().__init__(func=func, **kw)
|
|
@@ -54,13 +84,39 @@ def with_resources(*resources: Union[str, Resource], use_wrapper_class: bool = T
|
|
|
54
84
|
def func_wrapper(func):
|
|
55
85
|
str_resources = [res_to_str(r) for r in resources]
|
|
56
86
|
if not use_wrapper_class:
|
|
57
|
-
func
|
|
87
|
+
existing = getattr(func, "resources") or []
|
|
88
|
+
func.resources = existing + str_resources
|
|
89
|
+
return func
|
|
90
|
+
|
|
91
|
+
if isinstance(func, MarkedFunction):
|
|
92
|
+
func.resources = func.resources + str_resources
|
|
58
93
|
return func
|
|
94
|
+
return MarkedFunction(func, resources=str_resources)
|
|
95
|
+
|
|
96
|
+
return func_wrapper
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def with_python_requirements(
|
|
100
|
+
*requirements: str,
|
|
101
|
+
force_rebuild: bool = False,
|
|
102
|
+
prefer_binary: bool = False,
|
|
103
|
+
pre_release: bool = False,
|
|
104
|
+
):
|
|
105
|
+
result_req = []
|
|
106
|
+
for req in requirements:
|
|
107
|
+
result_req.extend(shlex.split(req))
|
|
59
108
|
|
|
109
|
+
def func_wrapper(func):
|
|
110
|
+
pack_item = PythonPackOptions(
|
|
111
|
+
requirements=requirements,
|
|
112
|
+
force_rebuild=force_rebuild,
|
|
113
|
+
prefer_binary=prefer_binary,
|
|
114
|
+
pre_release=pre_release,
|
|
115
|
+
)
|
|
60
116
|
if isinstance(func, MarkedFunction):
|
|
61
|
-
func.
|
|
117
|
+
func.pythonpacks.append(pack_item)
|
|
62
118
|
return func
|
|
63
|
-
return MarkedFunction(func,
|
|
119
|
+
return MarkedFunction(func, pythonpacks=[pack_item])
|
|
64
120
|
|
|
65
121
|
return func_wrapper
|
|
66
122
|
|
|
@@ -72,3 +128,7 @@ def get_udf_resources(
|
|
|
72
128
|
func: Callable,
|
|
73
129
|
) -> List[Union[Resource, str]]:
|
|
74
130
|
return getattr(func, "resources", None) or []
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def get_udf_pythonpacks(func: Callable) -> List[PythonPackOptions]:
|
|
134
|
+
return getattr(func, "pythonpacks", None) or []
|
maxframe/utils.py
CHANGED
|
@@ -1106,3 +1106,9 @@ def get_python_tag():
|
|
|
1106
1106
|
# todo add implementation suffix for non-GIL tags when PEP703 is ready
|
|
1107
1107
|
version_info = sys.version_info
|
|
1108
1108
|
return f"cp{version_info[0]}{version_info[1]}"
|
|
1109
|
+
|
|
1110
|
+
|
|
1111
|
+
def get_item_if_scalar(val: Any) -> Any:
|
|
1112
|
+
if isinstance(val, np.ndarray) and val.shape == ():
|
|
1113
|
+
return val.item()
|
|
1114
|
+
return val
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: maxframe
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.0b5
|
|
4
4
|
Summary: MaxFrame operator-based data analyze framework
|
|
5
5
|
Requires-Dist: numpy >=1.19.0
|
|
6
6
|
Requires-Dist: pandas >=1.0.0
|
|
7
|
-
Requires-Dist: pyodps >=0.11.
|
|
7
|
+
Requires-Dist: pyodps >=0.11.6.1
|
|
8
8
|
Requires-Dist: scipy >=1.0
|
|
9
9
|
Requires-Dist: pyarrow >=1.0.0
|
|
10
10
|
Requires-Dist: msgpack >=1.0.0
|
|
@@ -1,44 +1,44 @@
|
|
|
1
|
-
maxframe-0.1.
|
|
2
|
-
maxframe-0.1.
|
|
3
|
-
maxframe-0.1.
|
|
4
|
-
maxframe-0.1.
|
|
1
|
+
maxframe-0.1.0b5.dist-info/RECORD,,
|
|
2
|
+
maxframe-0.1.0b5.dist-info/WHEEL,sha256=r7U64H7df5k5VoE41bE2otJ6YmrMps4wUd2_S2hHvHQ,115
|
|
3
|
+
maxframe-0.1.0b5.dist-info/top_level.txt,sha256=64x-fc2q59c_vXwNUkehyjF1vb8JWqFSdYmUqIFqoTM,31
|
|
4
|
+
maxframe-0.1.0b5.dist-info/METADATA,sha256=cnNixA7GoFVbe5XauylcYj_NQVLErMPn_5uoJnSO5w8,3045
|
|
5
5
|
maxframe_client/conftest.py,sha256=7cwy2sFy5snEaxvtMvxfYFUnG6WtYC_9XxVrwJxOpcU,643
|
|
6
6
|
maxframe_client/__init__.py,sha256=3b-z0oFVVwtIzVFBxOb9pw7gz4IhTSh4FiHtVgnxS4Q,724
|
|
7
|
-
maxframe_client/fetcher.py,sha256=
|
|
7
|
+
maxframe_client/fetcher.py,sha256=p2hsVE2ihvhtuVCAaJhKccCiebl5f6BBPN6KIXOO0jo,8949
|
|
8
8
|
maxframe_client/clients/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
|
|
9
9
|
maxframe_client/clients/spe.py,sha256=ArZMNQ7olicI4O1JO7CyRP7-hb60DF71ZKCTO0N39uE,3593
|
|
10
10
|
maxframe_client/clients/framedriver.py,sha256=Rn09529D2qBTgNGc0oCY0l7b3FgzT87TqS1nujGQaHw,4463
|
|
11
|
-
maxframe_client/tests/test_session.py,sha256=
|
|
11
|
+
maxframe_client/tests/test_session.py,sha256=0G522Ia05fFuBJv6b6UTIxTNot3QK1Kjte6Xiimhouc,7884
|
|
12
12
|
maxframe_client/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
|
|
13
13
|
maxframe_client/tests/test_fetcher.py,sha256=q7kYCznM6WSxx9TCbHrxs7Zy1L2a5zu9D-Pi1XNgQzg,3516
|
|
14
|
-
maxframe_client/session/task.py,sha256=
|
|
14
|
+
maxframe_client/session/task.py,sha256=ec6AWeXBe_Q-Lats5ZW4V4J9qWokeq-ccBiAulCXpF8,10165
|
|
15
15
|
maxframe_client/session/graph.py,sha256=nwILNOIVaIf4E3xWffTAAlRsKRYU_zGW3oVO10du8Xw,4351
|
|
16
16
|
maxframe_client/session/__init__.py,sha256=KPqhSlAJiuUz8TC-z5o7mHDVXzLSqWwrZ33zNni7piY,832
|
|
17
17
|
maxframe_client/session/consts.py,sha256=R37BxDF3kgCy0qmDdwLaH5jB7mb7SzfYV6g9yHBKAwk,1344
|
|
18
|
-
maxframe_client/session/odps.py,sha256=
|
|
18
|
+
maxframe_client/session/odps.py,sha256=wIclE3cV2Awled2UiLUwATxsjc_fWFfbaW1UUcrmHCU,17526
|
|
19
19
|
maxframe_client/session/tests/test_task.py,sha256=lDdw3gToaM3xSaRXEmHUoAo2h0id7t4v_VvpdKxQAao,3279
|
|
20
20
|
maxframe_client/session/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
|
|
21
21
|
maxframe/_utils.pyx,sha256=I4kmfhNr-xXK2ak22dr4Vwahzn-JmTaYctbL3y9_UBQ,17017
|
|
22
22
|
maxframe/conftest.py,sha256=ZzwKhGp7LAVpzQYJkniwIUQqIegcaDQAhyzDyU2qld4,4264
|
|
23
|
-
maxframe/opcodes.py,sha256=
|
|
23
|
+
maxframe/opcodes.py,sha256=1Da6d3a82mecIHmnyZve4gSH_KN-q3Snl0nSygue2Ng,10191
|
|
24
24
|
maxframe/env.py,sha256=_K499f7giN7Iu9f39iI9p_naaEDoJ0rx8dInbzqFOVI,1402
|
|
25
25
|
maxframe/mixin.py,sha256=HBAeWYGb7N6ZIgkA-YpkKiSY1GetcEVNTuMb0ieznBs,3524
|
|
26
26
|
maxframe/protocol.py,sha256=LjjE6iw0ZVx82tBMbff4izkGuiJxRG0MTOaPYYpRL10,14190
|
|
27
|
-
maxframe/_utils.cpython-310-darwin.so,sha256=
|
|
28
|
-
maxframe/session.py,sha256=
|
|
29
|
-
maxframe/__init__.py,sha256=
|
|
30
|
-
maxframe/utils.py,sha256=
|
|
27
|
+
maxframe/_utils.cpython-310-darwin.so,sha256=YQC1CZBWBml2uaY12FrudKtGrM53xq40iN5m6D8VXOA,846464
|
|
28
|
+
maxframe/session.py,sha256=o2jp5NQP1pVMkp4AujN_3DB1HffZ0rMd5EbYZiLJxls,36205
|
|
29
|
+
maxframe/__init__.py,sha256=SqTFS_1o2HDuVY1mhS0ELlqDuM-biwM_MN0EYGkJLf0,1004
|
|
30
|
+
maxframe/utils.py,sha256=b4CvX6vh3AXmQKHQaxYIDtebJ4l6hMrTO_HqETwkVG0,34244
|
|
31
31
|
maxframe/extension.py,sha256=4IzF9sPyaRoAzLud0iDLooOgcyq4QunXH0ki3q9Hn8I,2374
|
|
32
32
|
maxframe/errors.py,sha256=nQZoLGdLJz-Uf9c2vUvQl08QMvRqtkBOhKfdPYRZA4o,690
|
|
33
|
-
maxframe/udf.py,sha256
|
|
33
|
+
maxframe/udf.py,sha256=-FzQCuV0GNyo3SH1r3rPQ3QMAydhYtE7KI8ff4Mv_PU,4288
|
|
34
34
|
maxframe/typing_.py,sha256=fzHETru3IOZAJwU9I7n_ib3wHuQRJ9XFVmAk7WpqkMo,1096
|
|
35
|
-
maxframe/codegen.py,sha256=
|
|
35
|
+
maxframe/codegen.py,sha256=B0y-bFXgze2CMUnAfrdOXsPrByyEAbq7pgio3mHl_j8,17516
|
|
36
36
|
maxframe/_utils.pxd,sha256=AhJ4vA_UqZqPshi5nvIZq1xgr80fhIVQ9dm5-UdkYJ8,1154
|
|
37
37
|
maxframe/dataframe/arrays.py,sha256=RWzimUcrds5CsIlPausfJAkLUjcktBSSXwdXyUNKEtU,28752
|
|
38
|
-
maxframe/dataframe/__init__.py,sha256=
|
|
39
|
-
maxframe/dataframe/core.py,sha256=
|
|
38
|
+
maxframe/dataframe/__init__.py,sha256=T08TZjNg2yWh7RmI0hX9PiBTlD5e9L_s3KnKsSLnBFA,2237
|
|
39
|
+
maxframe/dataframe/core.py,sha256=b27b0XomeqISCPxltavESlueTTYEXF5wjtrTKEZtld4,74413
|
|
40
40
|
maxframe/dataframe/initializer.py,sha256=4BpZJB8bbyFnABUYWBrk_qzzrogEsWgFuU21Ma9IsjY,10264
|
|
41
|
-
maxframe/dataframe/utils.py,sha256=
|
|
41
|
+
maxframe/dataframe/utils.py,sha256=uaRpQp5Lciow-9UBDnuUacYvi4erJgyvTDnQziRDkFU,44272
|
|
42
42
|
maxframe/dataframe/operators.py,sha256=T7Ik1shfoyrMZ1x0wHXG26bsod1_YjMGQgGAFNpH6k0,7871
|
|
43
43
|
maxframe/dataframe/statistics/corr.py,sha256=r9EwD6mqvlm3loEXr3ElS4XIr2NETfmTjC9V1fh4GW8,9495
|
|
44
44
|
maxframe/dataframe/statistics/quantile.py,sha256=eH-kAg-SZ5JdI8K3P6p9gNeDu4cbJaUC3Dt-zoCoTv0,11713
|
|
@@ -49,15 +49,17 @@ maxframe/dataframe/misc/accessor.py,sha256=QXjV5Q6LW0KNs67X5h2bCBhwHV8R7riP6TmTY
|
|
|
49
49
|
maxframe/dataframe/misc/describe.py,sha256=Qvmv_5rj-fk_ABcS5h8IxNHY-EAPB1rG446LiB1MRgU,4387
|
|
50
50
|
maxframe/dataframe/misc/astype.py,sha256=OkGDO6PBZli46m0kKq0BOO2OOwJWEyatzeBFV_mnKz8,7797
|
|
51
51
|
maxframe/dataframe/misc/to_numeric.py,sha256=gG5fxEtC71SkDeHaJ0Vd1JZrIM8gMHGFaiwyFtMwbWs,6262
|
|
52
|
+
maxframe/dataframe/misc/case_when.py,sha256=CqR8YRc7PKja3dUdPxe8nd7oAcZrvHAPJvdzlj9wZT0,4851
|
|
52
53
|
maxframe/dataframe/misc/check_monotonic.py,sha256=apHehWNRUTuzux3hI4JflWiuvpaeEFwtpPTWVide7wU,2422
|
|
53
54
|
maxframe/dataframe/misc/datetimes.py,sha256=fgLczaX-yDdairnNW1EsabEzoioP5gUsO0ukYANisLE,2503
|
|
54
55
|
maxframe/dataframe/misc/string_.py,sha256=eG4uMA6ScB0Pfby6pBLH24Ln01Aai2aAn-hMyRM5YFo,8101
|
|
55
56
|
maxframe/dataframe/misc/cut.py,sha256=aB8iuk6k2W3G2S3j0jPsryvBhOmaQJSZGIpZ5u1rWbs,13905
|
|
56
57
|
maxframe/dataframe/misc/get_dummies.py,sha256=q4f-1RCw-WWz53bT2FseYyzBNV3pDJTKNBKFx0dd888,7087
|
|
57
|
-
maxframe/dataframe/misc/__init__.py,sha256=
|
|
58
|
+
maxframe/dataframe/misc/__init__.py,sha256=U1p-l0iC1dbh7WLv5kVPelXqK7tb00LJvtu_H7GdtGA,5267
|
|
58
59
|
maxframe/dataframe/misc/qcut.py,sha256=De8kM1FfOWMEDvRNWRaupSHBRGLaOVrtGbLmtHAZvOg,3737
|
|
59
60
|
maxframe/dataframe/misc/pct_change.py,sha256=dNvj5VDPndRqc8lPweWcTiSz4p4oSibeQ2yxCVusLpI,4586
|
|
60
61
|
maxframe/dataframe/misc/duplicated.py,sha256=JPtBDLaxlVUqFBU3U3bqGnKQNaOcKwS5sdVfBFJt-Uo,8534
|
|
62
|
+
maxframe/dataframe/misc/pivot_table.py,sha256=YoKjjtOw2Z6KU3q3opAFag1gICz5Kc_VIcKNs3ur62A,9520
|
|
61
63
|
maxframe/dataframe/misc/map.py,sha256=RbUi0xPzT0hnsKid5RA5uHYyTeBnpzXlukR48Ntooxk,8149
|
|
62
64
|
maxframe/dataframe/misc/memory_usage.py,sha256=JN4x0ObjG7JP84ZMLRaQR8_-Eplg2_YxfJgi7XGM310,7897
|
|
63
65
|
maxframe/dataframe/misc/isin.py,sha256=POBR3Gvwlu2TH1AWWvKW12XOlu8HF2dJuBCBbOHgq74,6931
|
|
@@ -74,8 +76,8 @@ maxframe/dataframe/misc/eval.py,sha256=-2svFR5uraqnzKeCS2bJRx5POHRxqST2lbk6qzee7
|
|
|
74
76
|
maxframe/dataframe/misc/value_counts.py,sha256=ds3EVCs0jL1hcTWREzPgb7-20ezcYSNvu9ZtCYqrvfk,5492
|
|
75
77
|
maxframe/dataframe/misc/select_dtypes.py,sha256=xEdLNun58kvp-Vbl3u2cr8kaXFG8GsIU2BMfo7tFCM8,3144
|
|
76
78
|
maxframe/dataframe/misc/drop.py,sha256=-RHIQWek2LUTK8lOILNpCWYftCY3-I3DHSP0XT4uHBc,13029
|
|
77
|
-
maxframe/dataframe/misc/apply.py,sha256=
|
|
78
|
-
maxframe/dataframe/misc/tests/test_misc.py,sha256=
|
|
79
|
+
maxframe/dataframe/misc/apply.py,sha256=BTszTrgGDNATweYogT_0_hKyPF1A2xYVQGF81DYQLb4,23366
|
|
80
|
+
maxframe/dataframe/misc/tests/test_misc.py,sha256=Loi_HGVRe248yL7DI-5dAPR5eaJtPIXdaodFh8-fsOU,15142
|
|
79
81
|
maxframe/dataframe/misc/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
|
|
80
82
|
maxframe/dataframe/datasource/index.py,sha256=D7PcfIWS-6Hv1-8o6adNesditOAa9kb4JOQK1WlqNDQ,4401
|
|
81
83
|
maxframe/dataframe/datasource/read_csv.py,sha256=cpp6kX0DVrqY0KticL1h11tckobSzz1gyK4Sf2puUjo,24140
|
|
@@ -83,10 +85,10 @@ maxframe/dataframe/datasource/from_index.py,sha256=2061zsQn-BhyHTT0X9tE0JK8vLxQU
|
|
|
83
85
|
maxframe/dataframe/datasource/dataframe.py,sha256=LxAKF4gBIHhnJQPuaAUdIEyMAq7HTfiEeNVls5n4I4A,2023
|
|
84
86
|
maxframe/dataframe/datasource/series.py,sha256=QcYiBNcR8jjH6vdO6l6H9F46KHmlBqVCTI2tv9eyZ9w,1909
|
|
85
87
|
maxframe/dataframe/datasource/__init__.py,sha256=C8EKsHTJi-1jvJUKIpZtMtsK-ZID3dtxL1voXnaltTs,640
|
|
86
|
-
maxframe/dataframe/datasource/read_odps_query.py,sha256=
|
|
88
|
+
maxframe/dataframe/datasource/read_odps_query.py,sha256=CP7krcDLWrjReaQVIzr4FF4pI54D02aE0UXFYY89DFE,9970
|
|
87
89
|
maxframe/dataframe/datasource/core.py,sha256=ozFmDgw1og7nK9_jU-u3tLEq9pNbitN-8w8XWdbKkJ0,2687
|
|
88
90
|
maxframe/dataframe/datasource/date_range.py,sha256=CDGpxDyjLwnb66j-MIiiTfXGXHGh5MLhEmj6x2riIlU,17244
|
|
89
|
-
maxframe/dataframe/datasource/read_odps_table.py,sha256=
|
|
91
|
+
maxframe/dataframe/datasource/read_odps_table.py,sha256=UThcVjHKvt3GtNNd4zSEee3m87pKt9ngeksTvQSfH9k,9172
|
|
90
92
|
maxframe/dataframe/datasource/read_parquet.py,sha256=9auOcy8snTxCOohgXZCUXfT_O39irdkBngZH5svgx0E,14531
|
|
91
93
|
maxframe/dataframe/datasource/from_tensor.py,sha256=4viuN5SLLye7Xeb8kouOpm-osoQ2yEovWTDNPQuW8gE,14727
|
|
92
94
|
maxframe/dataframe/datasource/from_records.py,sha256=WBYouYyg7m_8NJdN-yUWSfJlIpm6DVP3IMfLXZFugyI,3442
|
|
@@ -153,7 +155,7 @@ maxframe/dataframe/reduction/unique.py,sha256=Dhghnf9mvgS8p-ETJUlhfJajBEyVZPBp1I
|
|
|
153
155
|
maxframe/dataframe/reduction/std.py,sha256=PYeh6-yton9vUI8D-dTlnXsWKCi-uDpop4UExmay7Ec,1355
|
|
154
156
|
maxframe/dataframe/reduction/str_concat.py,sha256=wbb4Y4ZDyou_ixR6ukJ8wQw74nzI05tToNWaWSirbSA,1657
|
|
155
157
|
maxframe/dataframe/reduction/__init__.py,sha256=ZadA_lXtrr1nQyb6vJrv_eb82bXiiiDHIn93F4O9-AU,4190
|
|
156
|
-
maxframe/dataframe/reduction/core.py,sha256=
|
|
158
|
+
maxframe/dataframe/reduction/core.py,sha256=ftArRDiA3mmj9AjBdR04BXmuHskZtgUqB5N8gJbHN9U,30395
|
|
157
159
|
maxframe/dataframe/reduction/all.py,sha256=h3Y04RJ-mTxAggX1-BIpuiyhNofQEkBzsAxKOX--kv0,1966
|
|
158
160
|
maxframe/dataframe/reduction/cummin.py,sha256=CA9wjhx_ZFfk6b3KbbDykWZiyXsfwLBUODKn36WuGug,1013
|
|
159
161
|
maxframe/dataframe/reduction/min.py,sha256=8Kh07yoTRWaCWGKxZ4a64_tlIljR8GsjlbwdsjNmPTw,1660
|
|
@@ -171,7 +173,7 @@ maxframe/dataframe/reduction/reduction_size.py,sha256=5wGc5cyuUllewvKPeJargbEjuz
|
|
|
171
173
|
maxframe/dataframe/reduction/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
|
|
172
174
|
maxframe/dataframe/reduction/tests/test_reduction.py,sha256=IMxAoGHMtz_54UNFO-_nmsNvq0-fuVZcxmf5sOAIyWs,17227
|
|
173
175
|
maxframe/dataframe/plotting/__init__.py,sha256=JShRu5lSUAfwgTFUMxQoIqiH1gg9H5cieoRyg4ybieA,1393
|
|
174
|
-
maxframe/dataframe/plotting/core.py,sha256
|
|
176
|
+
maxframe/dataframe/plotting/core.py,sha256=kvAM1ikU4SsFhvhmpOTQVTs2EX4ypAgNnjD9Jqy3xyA,2233
|
|
175
177
|
maxframe/dataframe/plotting/tests/test_plotting.py,sha256=IKAOcG0LotMGZCrvDzWiHmoFXqtyrVDn87eIOdyg6Ho,4118
|
|
176
178
|
maxframe/dataframe/plotting/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
|
|
177
179
|
maxframe/dataframe/window/aggregation.py,sha256=ncdJavwZcj6LWfF_W9wWUWAIjOxu7DOEEW16y5EQCF4,3794
|
|
@@ -255,7 +257,19 @@ maxframe/dataframe/indexing/reindex.py,sha256=mrDBxDZwbaqhjwy_fWGRtWI6MhJ6O1sR1A
|
|
|
255
257
|
maxframe/dataframe/indexing/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
|
|
256
258
|
maxframe/dataframe/indexing/tests/test_indexing.py,sha256=AcvF_exYVA6pf2giNzSAYfjAODbf207UtKSpTG68ssQ,15611
|
|
257
259
|
maxframe/learn/__init__.py,sha256=ii-8fJO-ubA3wRej1S1jXsb62zrS5MaAWc2gqQAAHFw,632
|
|
260
|
+
maxframe/learn/utils/__init__.py,sha256=zpkZ-HBwv91vyMeBVs59J_u0uVQCXUkEzo_pW2h9q00,646
|
|
261
|
+
maxframe/learn/utils/core.py,sha256=_tskQMfdvr5VAV68fDzM0DSVF2S7SMDnNXA8j_gHmGI,1029
|
|
258
262
|
maxframe/learn/contrib/__init__.py,sha256=WvCqsncQZKtDURfK8ma9GoPHVVK02yW2ueom_kBMl44,632
|
|
263
|
+
maxframe/learn/contrib/utils.py,sha256=k5jlwY8pru_PD8olOBOsMcBLeo31NMx53TjJmSvbvy8,1662
|
|
264
|
+
maxframe/learn/contrib/xgboost/classifier.py,sha256=IXNuAGtJMfZstcQoOyWUogZQsRvzRsBGp-gge-mqQsA,2936
|
|
265
|
+
maxframe/learn/contrib/xgboost/dmatrix.py,sha256=uZI3KxXZ7fa1SQ8dRs_ZTFpyNNt0ZmM4SR-WRkIvkjk,4856
|
|
266
|
+
maxframe/learn/contrib/xgboost/predict.py,sha256=yMV_wF3Cbx1PdZOjoy4mkVYnTQApmaC3ORK_9J8KicQ,4664
|
|
267
|
+
maxframe/learn/contrib/xgboost/__init__.py,sha256=kbYrokjDXjMBqRka5NTkUjU5zrNUoT_TkTyuW9G5s-0,899
|
|
268
|
+
maxframe/learn/contrib/xgboost/core.py,sha256=uilXCgjYuDKpxLNGkB5g-TDGijJ6EqnEvfXoEFC0JNQ,5194
|
|
269
|
+
maxframe/learn/contrib/xgboost/train.py,sha256=YkNTzyaxj_s9G-F2h0GX5L12PkJwOpcaTJX_R6DCQMQ,3958
|
|
270
|
+
maxframe/learn/contrib/xgboost/regressor.py,sha256=vAQHM2RQ3hKhQz3cqQ0twXpIDAkVOS9u4ivsQ5hX5dY,2532
|
|
271
|
+
maxframe/learn/contrib/xgboost/tests/test_core.py,sha256=0Q3A1EFW30vsQLl7bSTTuw-ldlLEO43ljVSJBb8j1LM,1406
|
|
272
|
+
maxframe/learn/contrib/xgboost/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
|
|
259
273
|
maxframe/learn/contrib/pytorch/run_script.py,sha256=Joh0gwukGf3kqfBYRxL34hQjbULT6s3vNnmBdJ6NZP0,3111
|
|
260
274
|
maxframe/learn/contrib/pytorch/__init__.py,sha256=koagbwLwH1VNLTWYx29ckatBChZ_f9wFcflB4tGYYwg,687
|
|
261
275
|
maxframe/learn/contrib/pytorch/run_function.py,sha256=oMDMVqDFQJ5d8DllZrqqRmBT2lx1cTmAv0hdQlO2GrY,3244
|
|
@@ -276,7 +290,7 @@ maxframe/core/entity/chunks.py,sha256=yNSLCWOpA_Z6aGr6ZI32dIJf3xPdRBWbvdsl8sTM3B
|
|
|
276
290
|
maxframe/core/graph/__init__.py,sha256=rnsXwW0ouh1f7SVtq73-PzLE-MBM6Op_0l6J7b7wGRE,821
|
|
277
291
|
maxframe/core/graph/entity.py,sha256=56gjXyDXN-TTPm3AQOxuRVQbb_fguKFDL_Xm7i95XEk,5559
|
|
278
292
|
maxframe/core/graph/core.pyx,sha256=ZJPx_MTOBMaX-6mns6tAiu-wrIBvRAKN44YAGTypJ1Y,15887
|
|
279
|
-
maxframe/core/graph/core.cpython-310-darwin.so,sha256=
|
|
293
|
+
maxframe/core/graph/core.cpython-310-darwin.so,sha256=Jksle-0LqVHeO1i52qxU5OMKF7K0we4PiTnn1ioEIDw,685568
|
|
280
294
|
maxframe/core/graph/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
|
|
281
295
|
maxframe/core/graph/tests/test_graph.py,sha256=kZfe_SfMOoHjEfXvtVtn0RjK4lOd-tFasxSpfL4G1WE,7462
|
|
282
296
|
maxframe/core/graph/builder/__init__.py,sha256=BV5tAiZ-bJE0FBGSCEnfqKNwBNHgvB9aSzEp0PY6SnA,702
|
|
@@ -295,7 +309,7 @@ maxframe/core/operator/fuse.py,sha256=0RGemF99gQCwV4aEk-K6T5KAGToO-487dFk8LyYDIZ
|
|
|
295
309
|
maxframe/core/operator/base.py,sha256=nxuSKjbBzDrItM9PGmFo8RLwParazu525jMLWj0kXkM,15251
|
|
296
310
|
maxframe/core/operator/tests/test_core.py,sha256=57aICnc5VLqdVK7icAORTWC81bSjBxeeVWIJcha9J_0,1691
|
|
297
311
|
maxframe/core/operator/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
|
|
298
|
-
maxframe/config/config.py,sha256=
|
|
312
|
+
maxframe/config/config.py,sha256=Uhwf6SqtMCRDVvif76lFEiGiYrIlUF97JbpVkOzHpic,13576
|
|
299
313
|
maxframe/config/validators.py,sha256=2m9MrkjDUFiU4PPaWIw8tjwMaOy8AYmuJFqVnnY8IMY,1615
|
|
300
314
|
maxframe/config/__init__.py,sha256=g5lN3nP2HTAXa6ExGxU1NwU1M9ulYPmAcsV-gU7nIW8,656
|
|
301
315
|
maxframe/config/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
|
|
@@ -304,13 +318,14 @@ maxframe/config/tests/test_config.py,sha256=Cx3buKli4F77zwJaB-vnUSlE9LUbt3ObHW38
|
|
|
304
318
|
maxframe/serialization/exception.py,sha256=TIlEDiuYwVgYkN4Se0ydXn2KzEJy4Yv8w2XwlTBqDdM,3027
|
|
305
319
|
maxframe/serialization/core.pxd,sha256=51XCBozW9j_E0sBgyYkHf5xNfqCMJb36Fwm_Pxv_rZs,1303
|
|
306
320
|
maxframe/serialization/pandas.py,sha256=D4_H4KjAl8DRQtODh9VBP94yB8ce1cYQUKrzXtn10KE,7147
|
|
321
|
+
maxframe/serialization/core.pyi,sha256=tU0i8Xtb5pW4NjSxylhCY3h5pDZx4uRmBb8Juk1CDdE,1922
|
|
307
322
|
maxframe/serialization/arrow.py,sha256=VnGxNLU9UV_cUPTze43bEFCIbYLAOZnp2pAwVJbAIzQ,3418
|
|
308
323
|
maxframe/serialization/__init__.py,sha256=9eSnoDww1uw2DAXEBBTB2atJQHzd-38XVxrCkoaypxA,921
|
|
309
324
|
maxframe/serialization/maxframe_objects.py,sha256=R9WEjbHL0Kr56OGkYDU9fcGi7gII6fGlXhi6IyihTsM,1365
|
|
310
325
|
maxframe/serialization/numpy.py,sha256=8_GSo45l_eNoMn4NAGEb9NLXY_9i4tf9KK4EzG0mKpA,3213
|
|
311
326
|
maxframe/serialization/scipy.py,sha256=hP0fAW0di9UgJrGtANB2S8hLDbFBtR8p5NDqAMt5rDI,2427
|
|
312
327
|
maxframe/serialization/core.pyx,sha256=AATN47RdBTq2zg7--3xX2VHyAZSvoAuYRt7B7gEgKPE,33984
|
|
313
|
-
maxframe/serialization/core.cpython-310-darwin.so,sha256
|
|
328
|
+
maxframe/serialization/core.cpython-310-darwin.so,sha256=hwKF5yZDqhf43dyL5qUPZLDzWFSRhgut5JX9R8vxzEc,1125296
|
|
314
329
|
maxframe/serialization/tests/test_serial.py,sha256=Wj_I6CBQMaOtE8WtqdUaBoU8FhBOihht6SfeHOJV-zU,12511
|
|
315
330
|
maxframe/serialization/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
|
|
316
331
|
maxframe/serialization/serializables/field.py,sha256=atVgX-9rsVG1fTev7vjQArVwIEaCRjXoSEjpQ3mh6bA,16015
|
|
@@ -320,14 +335,14 @@ maxframe/serialization/serializables/field_type.py,sha256=Feh09hu8XyaxS5MaJ4za_p
|
|
|
320
335
|
maxframe/serialization/serializables/tests/test_field_type.py,sha256=T3ebXbUkKveC9Pq1nIl85e4eYascFeJ52d0REHbz5jo,4381
|
|
321
336
|
maxframe/serialization/serializables/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
|
|
322
337
|
maxframe/serialization/serializables/tests/test_serializable.py,sha256=nwdN7B2xnI_Bh-s90TyjPvyFFVWOE9JVBqm4bdwYZ6o,8243
|
|
323
|
-
maxframe/odpsio/tableio.py,sha256=
|
|
324
|
-
maxframe/odpsio/arrow.py,sha256=
|
|
338
|
+
maxframe/odpsio/tableio.py,sha256=TiXZViZMqWq_RaSCCcw8EJuj7ymsKe7chIkUthrnS7I,10148
|
|
339
|
+
maxframe/odpsio/arrow.py,sha256=5GyAOaQLY_UeJS1CL98zNQpLMw5OfPrcI_YHP-G5Bas,3793
|
|
325
340
|
maxframe/odpsio/__init__.py,sha256=HcxZsE4hRwbhtE-ZXhDWZMmQlv-2dOTvQq2NajhGEo4,799
|
|
326
341
|
maxframe/odpsio/volumeio.py,sha256=b2SQqusgrtkJJ6uMjnFv5s56XjchF8F4lLTTSHynRMc,3743
|
|
327
342
|
maxframe/odpsio/schema.py,sha256=aXvK_1BSwttuUyROyIa_HNHohLZBp7LrW9VXzHPGXyY,12115
|
|
328
343
|
maxframe/odpsio/tests/test_tableio.py,sha256=5nKq8I8Pzrfl89BjIIGyrvqPRiXdejTcYCtd-R2vTAo,4653
|
|
329
344
|
maxframe/odpsio/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
|
|
330
|
-
maxframe/odpsio/tests/test_schema.py,sha256=
|
|
345
|
+
maxframe/odpsio/tests/test_schema.py,sha256=XTixGKHOWiprp5PZVOo2q9h4-HAUPL8vTUmoeXQDIiM,11958
|
|
331
346
|
maxframe/odpsio/tests/test_arrow.py,sha256=SQ9EmI9_VOOC8u6Rg6nh3IPC2fPbLvJ9HwtpMNDRhL8,3106
|
|
332
347
|
maxframe/odpsio/tests/test_volumeio.py,sha256=UEqFANuPKyFtlIh2JNi-LoixH52bxsgHdxu3himnEvs,3022
|
|
333
348
|
maxframe/tests/test_utils.py,sha256=xaAoURr5NOJUTY0XVa2H8qOStcEH5UQSXItkatHFxFE,11977
|
|
@@ -340,7 +355,7 @@ maxframe/lib/version.py,sha256=yQ6HkDOvU9X1rpI49auh-qku2g7gIiztgEH6v1urOrk,18321
|
|
|
340
355
|
maxframe/lib/compression.py,sha256=k9DSrl_dNBsn5azLjBdL5B4WZ6eNvmCrdMbcF1G7JSc,1442
|
|
341
356
|
maxframe/lib/__init__.py,sha256=CzfbLNqqm1yR1i6fDwCd4h1ptuKVDbURFVCb0ra7QNc,642
|
|
342
357
|
maxframe/lib/functools_compat.py,sha256=PMSkct9GIbzq-aBwTnggrOLNfLh4xQnYTIFMPblzCUA,2616
|
|
343
|
-
maxframe/lib/mmh3.cpython-310-darwin.so,sha256=
|
|
358
|
+
maxframe/lib/mmh3.cpython-310-darwin.so,sha256=zU0-yFk7Dr4DaPBvU1lQQnO_aoJ-b10n8RTOpp6EWFc,119784
|
|
344
359
|
maxframe/lib/mmh3_src/mmh3module.cpp,sha256=9J9eA42eKWTl546fvfQPNuIM3B2jpWSADpgIw3tr2jg,11604
|
|
345
360
|
maxframe/lib/mmh3_src/MurmurHash3.h,sha256=lg5uXUFyMBge2BWRn0FgrqaCFCMfDWoTXD4PQtjHrMA,1263
|
|
346
361
|
maxframe/lib/mmh3_src/MurmurHash3.cpp,sha256=kgrteG44VSftwp5hhD7pyTENDRU9wI_DqLD1493-bP0,8096
|
|
@@ -387,7 +402,7 @@ maxframe/lib/tblib/__init__.py,sha256=c4aVldbxJdS-bFsSDcmDQy_mW0qAcMrb4pHS2tjYhY
|
|
|
387
402
|
maxframe/lib/tblib/cpython.py,sha256=FQ0f6WTQyQHoMRhgPqrA0y0Ygxlbj5IC53guxA4h9Cw,2418
|
|
388
403
|
maxframe/lib/tblib/decorators.py,sha256=bcllK3kVuPnj6SNZGmlJGxTK0ovdt7TJDXrhA4UE5sQ,1063
|
|
389
404
|
maxframe/tensor/array_utils.py,sha256=259vG4SjyhiheARCZeEnfJdZjoojyrELn41oRcyAELs,4943
|
|
390
|
-
maxframe/tensor/__init__.py,sha256
|
|
405
|
+
maxframe/tensor/__init__.py,sha256=BJgqcCnG0jgPBafI4Aa01gHL4nagu6l7kmlry7u8Zm8,3519
|
|
391
406
|
maxframe/tensor/core.py,sha256=Ojxaf5b8sJ6ZZGezyFHQJ5XsSpUrBOnZgFeUQgpVJpI,21914
|
|
392
407
|
maxframe/tensor/utils.py,sha256=bwVN0iuVic1tpFai6Hk-1tQLqckQ2IYS7yZKMTcOU1I,22914
|
|
393
408
|
maxframe/tensor/operators.py,sha256=iGkDIRz152gXrPb5JbqOvXngpq3QaCg-aNO4gHZPLN0,3461
|
|
@@ -401,13 +416,13 @@ maxframe/tensor/reshape/tests/test_reshape.py,sha256=MkXN_ibLMNn_4vc8Jsq39icqTIY
|
|
|
401
416
|
maxframe/tensor/datasource/from_dataframe.py,sha256=X6cfwLT__VGZLoy3U3tjehPnO43CnyN4kK4edDtLFrA,2503
|
|
402
417
|
maxframe/tensor/datasource/zeros.py,sha256=vMJ44GeCPOxtnj-NnKvkbb4u3U_yxcEhEJa1-ADt1fM,5672
|
|
403
418
|
maxframe/tensor/datasource/from_dense.py,sha256=N2FVRhU9uh-w1W8W9zvpTQVxGgskSdHZbVh_rJjC6QA,1607
|
|
404
|
-
maxframe/tensor/datasource/scalar.py,sha256=
|
|
419
|
+
maxframe/tensor/datasource/scalar.py,sha256=GzKdbBLFdq9aeWrrtZ3BT5514fBAi9A-QJ5M-EmfUX0,1156
|
|
405
420
|
maxframe/tensor/datasource/empty.py,sha256=e6XYQvFsyfnGqIrU216_Q1M7cYxwkQSOoZiEhtOT8nc,5850
|
|
406
421
|
maxframe/tensor/datasource/__init__.py,sha256=qU_GFAuMa_U_zrV6HcE9fsgO16EXVsqhh9j5Fxe6HFY,1146
|
|
407
422
|
maxframe/tensor/datasource/core.py,sha256=VpjxIcWp_O00smfvDsRIQ48d3HOIJ81hkn78S3H2n_U,3411
|
|
408
423
|
maxframe/tensor/datasource/full.py,sha256=2oaPGJDbnIz9th74KtGYv-l4cienh679dGueJCi5T08,6276
|
|
409
424
|
maxframe/tensor/datasource/arange.py,sha256=jqUF1P1Smu2Bl1j5uOe_MM1EPft8heJu8WSmDmyFiDc,5476
|
|
410
|
-
maxframe/tensor/datasource/array.py,sha256=
|
|
425
|
+
maxframe/tensor/datasource/array.py,sha256=RmSX0OZrtPPfUNxJwRef8uJlTt76QF0xAeU3U8Wu2kE,13004
|
|
411
426
|
maxframe/tensor/datasource/from_sparse.py,sha256=ki_cSjqrUccChg2uqlXy1xF10c4YANA4QP_UDl_LZnA,1546
|
|
412
427
|
maxframe/tensor/datasource/ones.py,sha256=ER4NJ53HuGlSsz-cRXz9YSwZkokdc2wkBKdB2G1BMX8,5009
|
|
413
428
|
maxframe/tensor/datasource/tests/test_datasource.py,sha256=SEyzSKnf0J6nytli1f2Lgu-U4lp6pMvbuyNM-90x9wQ,7653
|
|
@@ -619,10 +634,12 @@ maxframe/tensor/random/tests/__init__.py,sha256=CzfbLNqqm1yR1i6fDwCd4h1ptuKVDbUR
|
|
|
619
634
|
maxframe/tensor/random/tests/test_random.py,sha256=2hGaik9A783PFuuxIrhCDZ2O-SYKwUh98LKPwf1psLs,4275
|
|
620
635
|
maxframe/tensor/base/astype.py,sha256=zeLfyWkuc1LxMIiIO6ghZ_enR7IPgq0Hy4O18h8JQPk,4394
|
|
621
636
|
maxframe/tensor/base/where.py,sha256=fMGBfppo4QToHX-B3rtTorstz83m8uWWhOQj6Qbu34U,4000
|
|
622
|
-
maxframe/tensor/base/
|
|
637
|
+
maxframe/tensor/base/unique.py,sha256=MFJMBZAGHLnyGL5RGClCzwVKeSb0v0l0hDJ2zgXuSTQ,6742
|
|
638
|
+
maxframe/tensor/base/__init__.py,sha256=FMnuIAaGUObPZpXBnN-9eTSmmdq0Cdb-VI_JNHKVy3Q,1079
|
|
623
639
|
maxframe/tensor/base/transpose.py,sha256=yDfr1vD6doNlsuKuU94daE0IgIwGMuyBfXhKauQ1UaM,3414
|
|
624
640
|
maxframe/tensor/base/ravel.py,sha256=pJkez1NHoLqVMBPm5aA8uMNFeWURTAJV_iU98Vqr-50,3173
|
|
625
641
|
maxframe/tensor/base/broadcast_to.py,sha256=p0hi128OWlj3lXmacvxMhZOjUCq8fiA3yjy9nESZXgE,2686
|
|
642
|
+
maxframe/tensor/base/atleast_1d.py,sha256=rpzzmdCgdz-IrL5WtFd31BPHaCS0V_wLuYJHjVo-vnE,1918
|
|
626
643
|
maxframe/tensor/base/tests/__init__.py,sha256=FEFOVLi3o2GpZoedTtLYvbie0eQBehJIjtCrWca2ZHw,596
|
|
627
644
|
maxframe/tensor/base/tests/test_base.py,sha256=syedkA0jMjevYouCL3vCMuWpJ-3hfOc28wKXqF_MMno,2843
|
|
628
645
|
maxframe/remote/run_script.py,sha256=Z1j662AwDF9sr-z1xnPTlrfSTixi2RBZjccXEilfpGA,3556
|
maxframe_client/fetcher.py
CHANGED
|
@@ -12,16 +12,21 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
+
import base64
|
|
16
|
+
import json
|
|
15
17
|
from abc import ABC, abstractmethod
|
|
16
18
|
from numbers import Integral
|
|
17
|
-
from typing import Any, Dict, List, Type, Union
|
|
19
|
+
from typing import Any, Dict, List, Optional, Type, Union
|
|
18
20
|
|
|
21
|
+
import pandas as pd
|
|
19
22
|
import pyarrow as pa
|
|
20
23
|
from odps import ODPS
|
|
21
24
|
from odps.models import ExternalVolume, PartedVolume
|
|
25
|
+
from odps.tunnel import TableTunnel
|
|
22
26
|
from tornado import httpclient
|
|
23
27
|
|
|
24
28
|
from maxframe.core import OBJECT_TYPE
|
|
29
|
+
from maxframe.dataframe.core import DATAFRAME_TYPE
|
|
25
30
|
from maxframe.lib import wrapped_pickle as pickle
|
|
26
31
|
from maxframe.odpsio import HaloTableIO, arrow_to_pandas, build_dataframe_table_meta
|
|
27
32
|
from maxframe.protocol import (
|
|
@@ -31,8 +36,9 @@ from maxframe.protocol import (
|
|
|
31
36
|
ResultInfo,
|
|
32
37
|
ResultType,
|
|
33
38
|
)
|
|
39
|
+
from maxframe.tensor.core import TENSOR_TYPE
|
|
34
40
|
from maxframe.typing_ import PandasObjectTypes, TileableType
|
|
35
|
-
from maxframe.utils import ToThreadMixin
|
|
41
|
+
from maxframe.utils import ToThreadMixin, deserialize_serializable
|
|
36
42
|
|
|
37
43
|
_result_fetchers: Dict[ResultType, Type["ResultFetcher"]] = dict()
|
|
38
44
|
|
|
@@ -52,6 +58,14 @@ class ResultFetcher(ABC):
|
|
|
52
58
|
def __init__(self, odps_entry: ODPS):
|
|
53
59
|
self._odps_entry = odps_entry
|
|
54
60
|
|
|
61
|
+
@abstractmethod
|
|
62
|
+
async def update_tileable_meta(
|
|
63
|
+
self,
|
|
64
|
+
tileable: TileableType,
|
|
65
|
+
info: ResultInfo,
|
|
66
|
+
) -> None:
|
|
67
|
+
raise NotImplementedError
|
|
68
|
+
|
|
55
69
|
@abstractmethod
|
|
56
70
|
async def fetch(
|
|
57
71
|
self,
|
|
@@ -66,6 +80,13 @@ class ResultFetcher(ABC):
|
|
|
66
80
|
class NullFetcher(ResultFetcher):
|
|
67
81
|
result_type = ResultType.NULL
|
|
68
82
|
|
|
83
|
+
async def update_tileable_meta(
|
|
84
|
+
self,
|
|
85
|
+
tileable: TileableType,
|
|
86
|
+
info: ResultInfo,
|
|
87
|
+
) -> None:
|
|
88
|
+
return
|
|
89
|
+
|
|
69
90
|
async def fetch(
|
|
70
91
|
self,
|
|
71
92
|
tileable: TileableType,
|
|
@@ -79,6 +100,40 @@ class NullFetcher(ResultFetcher):
|
|
|
79
100
|
class ODPSTableFetcher(ToThreadMixin, ResultFetcher):
|
|
80
101
|
result_type = ResultType.ODPS_TABLE
|
|
81
102
|
|
|
103
|
+
def _get_table_comment(self, table_name: str) -> Optional[str]:
|
|
104
|
+
table = self._odps_entry.get_table(table_name)
|
|
105
|
+
return getattr(table, "comment", None)
|
|
106
|
+
|
|
107
|
+
async def update_tileable_meta(
|
|
108
|
+
self,
|
|
109
|
+
tileable: TileableType,
|
|
110
|
+
info: ODPSTableResultInfo,
|
|
111
|
+
) -> None:
|
|
112
|
+
if isinstance(tileable, DATAFRAME_TYPE) and tileable.dtypes is None:
|
|
113
|
+
tb_comment = await self.to_thread(
|
|
114
|
+
self._get_table_comment, info.full_table_name
|
|
115
|
+
)
|
|
116
|
+
if tb_comment: # pragma: no branch
|
|
117
|
+
comment_data = json.loads(tb_comment)
|
|
118
|
+
|
|
119
|
+
table_meta: DataFrameTableMeta = deserialize_serializable(
|
|
120
|
+
base64.b64decode(comment_data["table_meta"])
|
|
121
|
+
)
|
|
122
|
+
tileable.refresh_from_table_meta(table_meta)
|
|
123
|
+
|
|
124
|
+
if tileable.shape and any(pd.isna(x) for x in tileable.shape):
|
|
125
|
+
part_specs = [None] if not info.partition_specs else info.partition_specs
|
|
126
|
+
tunnel = TableTunnel(self._odps_entry)
|
|
127
|
+
total_records = 0
|
|
128
|
+
for part_spec in part_specs:
|
|
129
|
+
session = tunnel.create_download_session(
|
|
130
|
+
info.full_table_name, part_spec
|
|
131
|
+
)
|
|
132
|
+
total_records += session.count
|
|
133
|
+
new_shape_list = list(tileable.shape)
|
|
134
|
+
new_shape_list[-1] = total_records
|
|
135
|
+
tileable.params = {"shape": tuple(new_shape_list)}
|
|
136
|
+
|
|
82
137
|
def _read_single_source(
|
|
83
138
|
self,
|
|
84
139
|
table_meta: DataFrameTableMeta,
|
|
@@ -149,6 +204,13 @@ class ODPSTableFetcher(ToThreadMixin, ResultFetcher):
|
|
|
149
204
|
class ODPSVolumeFetcher(ToThreadMixin, ResultFetcher):
|
|
150
205
|
result_type = ResultType.ODPS_VOLUME
|
|
151
206
|
|
|
207
|
+
async def update_tileable_meta(
|
|
208
|
+
self,
|
|
209
|
+
tileable: TileableType,
|
|
210
|
+
info: ODPSVolumeResultInfo,
|
|
211
|
+
) -> None:
|
|
212
|
+
return
|
|
213
|
+
|
|
152
214
|
async def _read_parted_volume_data(
|
|
153
215
|
self, volume: PartedVolume, partition: str, file_name: str
|
|
154
216
|
) -> bytes:
|
|
@@ -197,6 +259,6 @@ class ODPSVolumeFetcher(ToThreadMixin, ResultFetcher):
|
|
|
197
259
|
info: ODPSVolumeResultInfo,
|
|
198
260
|
indexes: List[Union[Integral, slice]],
|
|
199
261
|
) -> Any:
|
|
200
|
-
if isinstance(tileable, OBJECT_TYPE):
|
|
262
|
+
if isinstance(tileable, (OBJECT_TYPE, TENSOR_TYPE)):
|
|
201
263
|
return await self._fetch_object(info)
|
|
202
264
|
raise NotImplementedError(f"Fetching {type(tileable)} not implemented")
|
maxframe_client/session/odps.py
CHANGED
|
@@ -84,6 +84,9 @@ class MaxFrameServiceCaller(metaclass=abc.ABCMeta):
|
|
|
84
84
|
def decref(self, tileable_keys: List[str]) -> None:
|
|
85
85
|
raise NotImplementedError
|
|
86
86
|
|
|
87
|
+
def get_logview_address(self, dag_id=None, hours=None) -> Optional[str]:
|
|
88
|
+
return None
|
|
89
|
+
|
|
87
90
|
|
|
88
91
|
class MaxFrameSession(ToThreadMixin, IsolatedAsyncSession):
|
|
89
92
|
_odps_entry: Optional[ODPS]
|
|
@@ -129,6 +132,7 @@ class MaxFrameSession(ToThreadMixin, IsolatedAsyncSession):
|
|
|
129
132
|
async def _init(self, _address: str):
|
|
130
133
|
session_info = await self.ensure_async_call(self._caller.create_session)
|
|
131
134
|
self._session_id = session_info.session_id
|
|
135
|
+
await self._show_logview_address()
|
|
132
136
|
|
|
133
137
|
def _upload_and_get_read_tileable(self, t: TileableType) -> Optional[TileableType]:
|
|
134
138
|
if (
|
|
@@ -142,7 +146,9 @@ class MaxFrameSession(ToThreadMixin, IsolatedAsyncSession):
|
|
|
142
146
|
if self._odps_entry.exist_table(table_meta.table_name):
|
|
143
147
|
self._odps_entry.delete_table(table_meta.table_name)
|
|
144
148
|
table_name = build_temp_table_name(self.session_id, t.key)
|
|
145
|
-
table_obj = self._odps_entry.create_table(
|
|
149
|
+
table_obj = self._odps_entry.create_table(
|
|
150
|
+
table_name, schema, lifecycle=options.session.temp_table_lifecycle
|
|
151
|
+
)
|
|
146
152
|
|
|
147
153
|
data = t.op.get_data()
|
|
148
154
|
batch_size = options.session.upload_batch_size
|
|
@@ -239,6 +245,8 @@ class MaxFrameSession(ToThreadMixin, IsolatedAsyncSession):
|
|
|
239
245
|
self._caller.submit_dag, tileable_graph, replaced_infos
|
|
240
246
|
)
|
|
241
247
|
|
|
248
|
+
await self._show_logview_address(dag_info.dag_id)
|
|
249
|
+
|
|
242
250
|
progress = Progress()
|
|
243
251
|
profiling = Profiling()
|
|
244
252
|
aio_task = asyncio.create_task(
|
|
@@ -294,6 +302,8 @@ class MaxFrameSession(ToThreadMixin, IsolatedAsyncSession):
|
|
|
294
302
|
|
|
295
303
|
for key, result_info in dag_info.tileable_to_result_infos.items():
|
|
296
304
|
t = key_to_tileables[key]
|
|
305
|
+
fetcher = get_fetcher_cls(result_info.result_type)(self._odps_entry)
|
|
306
|
+
await fetcher.update_tileable_meta(t, result_info)
|
|
297
307
|
self._tileable_to_infos[t] = result_info
|
|
298
308
|
|
|
299
309
|
def _get_data_tileable_and_indexes(
|
|
@@ -388,6 +398,25 @@ class MaxFrameSession(ToThreadMixin, IsolatedAsyncSession):
|
|
|
388
398
|
async def get_mutable_tensor(self, name: str):
|
|
389
399
|
raise NotImplementedError
|
|
390
400
|
|
|
401
|
+
async def get_logview_address(self, hours=None) -> Optional[str]:
|
|
402
|
+
return await self.get_dag_logview_address(None, hours)
|
|
403
|
+
|
|
404
|
+
async def get_dag_logview_address(self, dag_id=None, hours=None) -> Optional[str]:
|
|
405
|
+
return await self.ensure_async_call(
|
|
406
|
+
self._caller.get_logview_address, dag_id, hours
|
|
407
|
+
)
|
|
408
|
+
|
|
409
|
+
async def _show_logview_address(self, dag_id=None, hours=None):
|
|
410
|
+
identity = f"Session ID: {self._session_id}"
|
|
411
|
+
if dag_id:
|
|
412
|
+
identity += f", DAG ID: {dag_id}"
|
|
413
|
+
|
|
414
|
+
logview_addr = await self.get_dag_logview_address(dag_id, hours)
|
|
415
|
+
if logview_addr:
|
|
416
|
+
logger.info("%s, Logview: %s", identity, logview_addr)
|
|
417
|
+
else:
|
|
418
|
+
logger.info("%s, Logview address does not exist", identity)
|
|
419
|
+
|
|
391
420
|
|
|
392
421
|
class MaxFrameRestCaller(MaxFrameServiceCaller):
|
|
393
422
|
_client: FrameDriverClient
|