maxframe 1.3.1__cp38-cp38-macosx_10_9_universal2.whl → 2.0.0b1__cp38-cp38-macosx_10_9_universal2.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of maxframe might be problematic. Click here for more details.

Files changed (639) hide show
  1. maxframe/_utils.cpython-38-darwin.so +0 -0
  2. maxframe/_utils.pyi +21 -0
  3. maxframe/_utils.pyx +4 -3
  4. maxframe/codegen/__init__.py +27 -0
  5. maxframe/{codegen.py → codegen/core.py} +49 -43
  6. maxframe/codegen/spe/__init__.py +16 -0
  7. maxframe/codegen/spe/core.py +307 -0
  8. maxframe/codegen/spe/dataframe/__init__.py +37 -0
  9. maxframe/codegen/spe/dataframe/accessors/__init__.py +15 -0
  10. maxframe/codegen/spe/dataframe/accessors/base.py +53 -0
  11. maxframe/codegen/spe/dataframe/accessors/dict_.py +194 -0
  12. maxframe/codegen/spe/dataframe/accessors/list_.py +80 -0
  13. maxframe/codegen/spe/dataframe/arithmetic.py +84 -0
  14. maxframe/codegen/spe/dataframe/datasource.py +181 -0
  15. maxframe/codegen/spe/dataframe/datastore.py +204 -0
  16. maxframe/codegen/spe/dataframe/extensions.py +63 -0
  17. maxframe/codegen/spe/dataframe/fetch.py +26 -0
  18. maxframe/codegen/spe/dataframe/groupby.py +224 -0
  19. maxframe/codegen/spe/dataframe/indexing.py +238 -0
  20. maxframe/codegen/spe/dataframe/merge.py +73 -0
  21. maxframe/codegen/spe/dataframe/misc.py +286 -0
  22. maxframe/codegen/spe/dataframe/missing.py +64 -0
  23. maxframe/codegen/spe/dataframe/reduction.py +160 -0
  24. maxframe/codegen/spe/dataframe/sort.py +83 -0
  25. maxframe/codegen/spe/dataframe/statistics.py +46 -0
  26. maxframe/codegen/spe/dataframe/tests/__init__.py +13 -0
  27. maxframe/codegen/spe/dataframe/tests/accessors/__init__.py +13 -0
  28. maxframe/codegen/spe/dataframe/tests/accessors/test_base.py +33 -0
  29. maxframe/codegen/spe/dataframe/tests/accessors/test_dict.py +310 -0
  30. maxframe/codegen/spe/dataframe/tests/accessors/test_list.py +137 -0
  31. maxframe/codegen/spe/dataframe/tests/indexing/__init__.py +13 -0
  32. maxframe/codegen/spe/dataframe/tests/indexing/conftest.py +58 -0
  33. maxframe/codegen/spe/dataframe/tests/indexing/test_getitem.py +124 -0
  34. maxframe/codegen/spe/dataframe/tests/indexing/test_iloc.py +76 -0
  35. maxframe/codegen/spe/dataframe/tests/indexing/test_indexing.py +39 -0
  36. maxframe/codegen/spe/dataframe/tests/indexing/test_rename.py +51 -0
  37. maxframe/codegen/spe/dataframe/tests/indexing/test_reset_index.py +88 -0
  38. maxframe/codegen/spe/dataframe/tests/indexing/test_sample.py +45 -0
  39. maxframe/codegen/spe/dataframe/tests/indexing/test_set_axis.py +45 -0
  40. maxframe/codegen/spe/dataframe/tests/indexing/test_set_index.py +41 -0
  41. maxframe/codegen/spe/dataframe/tests/indexing/test_setitem.py +46 -0
  42. maxframe/codegen/spe/dataframe/tests/misc/__init__.py +13 -0
  43. maxframe/codegen/spe/dataframe/tests/misc/test_apply.py +133 -0
  44. maxframe/codegen/spe/dataframe/tests/misc/test_drop_duplicates.py +92 -0
  45. maxframe/codegen/spe/dataframe/tests/misc/test_misc.py +234 -0
  46. maxframe/codegen/spe/dataframe/tests/missing/__init__.py +13 -0
  47. maxframe/codegen/spe/dataframe/tests/missing/test_checkna.py +94 -0
  48. maxframe/codegen/spe/dataframe/tests/missing/test_dropna.py +50 -0
  49. maxframe/codegen/spe/dataframe/tests/missing/test_fillna.py +94 -0
  50. maxframe/codegen/spe/dataframe/tests/missing/test_replace.py +45 -0
  51. maxframe/codegen/spe/dataframe/tests/test_arithmetic.py +73 -0
  52. maxframe/codegen/spe/dataframe/tests/test_datasource.py +184 -0
  53. maxframe/codegen/spe/dataframe/tests/test_datastore.py +200 -0
  54. maxframe/codegen/spe/dataframe/tests/test_extensions.py +88 -0
  55. maxframe/codegen/spe/dataframe/tests/test_groupby.py +225 -0
  56. maxframe/codegen/spe/dataframe/tests/test_merge.py +400 -0
  57. maxframe/codegen/spe/dataframe/tests/test_reduction.py +104 -0
  58. maxframe/codegen/spe/dataframe/tests/test_sort.py +159 -0
  59. maxframe/codegen/spe/dataframe/tests/test_statistics.py +70 -0
  60. maxframe/codegen/spe/dataframe/tests/test_tseries.py +29 -0
  61. maxframe/codegen/spe/dataframe/tests/test_value_counts.py +60 -0
  62. maxframe/codegen/spe/dataframe/tests/test_window.py +69 -0
  63. maxframe/codegen/spe/dataframe/tseries.py +46 -0
  64. maxframe/codegen/spe/dataframe/udf.py +62 -0
  65. maxframe/codegen/spe/dataframe/value_counts.py +31 -0
  66. maxframe/codegen/spe/dataframe/window.py +65 -0
  67. maxframe/codegen/spe/learn/__init__.py +15 -0
  68. maxframe/codegen/spe/learn/contrib/__init__.py +15 -0
  69. maxframe/codegen/spe/learn/contrib/lightgbm.py +160 -0
  70. maxframe/codegen/spe/learn/contrib/models.py +41 -0
  71. maxframe/codegen/spe/learn/contrib/pytorch.py +49 -0
  72. maxframe/codegen/spe/learn/contrib/tests/__init__.py +13 -0
  73. maxframe/codegen/spe/learn/contrib/tests/test_lightgbm.py +123 -0
  74. maxframe/codegen/spe/learn/contrib/tests/test_models.py +41 -0
  75. maxframe/codegen/spe/learn/contrib/tests/test_pytorch.py +53 -0
  76. maxframe/codegen/spe/learn/contrib/tests/test_xgboost.py +98 -0
  77. maxframe/codegen/spe/learn/contrib/xgboost.py +152 -0
  78. maxframe/codegen/spe/learn/metrics/__init__.py +15 -0
  79. maxframe/codegen/spe/learn/metrics/_classification.py +120 -0
  80. maxframe/codegen/spe/learn/metrics/tests/__init__.py +13 -0
  81. maxframe/codegen/spe/learn/metrics/tests/test_classification.py +93 -0
  82. maxframe/codegen/spe/learn/model_selection/__init__.py +13 -0
  83. maxframe/codegen/spe/learn/model_selection/tests/__init__.py +13 -0
  84. maxframe/codegen/spe/learn/model_selection/tests/test_split.py +41 -0
  85. maxframe/codegen/spe/learn/preprocessing/__init__.py +15 -0
  86. maxframe/codegen/spe/learn/preprocessing/_data.py +37 -0
  87. maxframe/codegen/spe/learn/preprocessing/_label.py +47 -0
  88. maxframe/codegen/spe/learn/preprocessing/tests/__init__.py +13 -0
  89. maxframe/codegen/spe/learn/preprocessing/tests/test_data.py +31 -0
  90. maxframe/codegen/spe/learn/preprocessing/tests/test_label.py +43 -0
  91. maxframe/codegen/spe/learn/utils/__init__.py +15 -0
  92. maxframe/codegen/spe/learn/utils/checks.py +55 -0
  93. maxframe/codegen/spe/learn/utils/multiclass.py +60 -0
  94. maxframe/codegen/spe/learn/utils/shuffle.py +85 -0
  95. maxframe/codegen/spe/learn/utils/sparsefuncs.py +35 -0
  96. maxframe/codegen/spe/learn/utils/tests/__init__.py +13 -0
  97. maxframe/codegen/spe/learn/utils/tests/test_checks.py +48 -0
  98. maxframe/codegen/spe/learn/utils/tests/test_multiclass.py +52 -0
  99. maxframe/codegen/spe/learn/utils/tests/test_shuffle.py +50 -0
  100. maxframe/codegen/spe/learn/utils/tests/test_sparsefuncs.py +34 -0
  101. maxframe/codegen/spe/learn/utils/tests/test_validation.py +44 -0
  102. maxframe/codegen/spe/learn/utils/validation.py +35 -0
  103. maxframe/codegen/spe/objects.py +26 -0
  104. maxframe/codegen/spe/remote.py +29 -0
  105. maxframe/codegen/spe/tensor/__init__.py +28 -0
  106. maxframe/codegen/spe/tensor/arithmetic.py +95 -0
  107. maxframe/codegen/spe/tensor/core.py +41 -0
  108. maxframe/codegen/spe/tensor/datasource.py +165 -0
  109. maxframe/codegen/spe/tensor/extensions.py +35 -0
  110. maxframe/codegen/spe/tensor/fetch.py +26 -0
  111. maxframe/codegen/spe/tensor/indexing.py +63 -0
  112. maxframe/codegen/spe/tensor/linalg.py +63 -0
  113. maxframe/codegen/spe/tensor/merge.py +31 -0
  114. maxframe/codegen/spe/tensor/misc.py +121 -0
  115. maxframe/codegen/spe/tensor/random.py +29 -0
  116. maxframe/codegen/spe/tensor/reduction.py +39 -0
  117. maxframe/codegen/spe/tensor/reshape.py +26 -0
  118. maxframe/codegen/spe/tensor/sort.py +42 -0
  119. maxframe/codegen/spe/tensor/special.py +35 -0
  120. maxframe/codegen/spe/tensor/statistics.py +24 -0
  121. maxframe/codegen/spe/tensor/tests/__init__.py +13 -0
  122. maxframe/codegen/spe/tensor/tests/test_arithmetic.py +103 -0
  123. maxframe/codegen/spe/tensor/tests/test_datasource.py +99 -0
  124. maxframe/codegen/spe/tensor/tests/test_extensions.py +37 -0
  125. maxframe/codegen/spe/tensor/tests/test_indexing.py +44 -0
  126. maxframe/codegen/spe/tensor/tests/test_linalg.py +38 -0
  127. maxframe/codegen/spe/tensor/tests/test_merge.py +28 -0
  128. maxframe/codegen/spe/tensor/tests/test_misc.py +94 -0
  129. maxframe/codegen/spe/tensor/tests/test_random.py +55 -0
  130. maxframe/codegen/spe/tensor/tests/test_reduction.py +65 -0
  131. maxframe/codegen/spe/tensor/tests/test_reshape.py +39 -0
  132. maxframe/codegen/spe/tensor/tests/test_sort.py +49 -0
  133. maxframe/codegen/spe/tensor/tests/test_special.py +28 -0
  134. maxframe/codegen/spe/tensor/tests/test_statistics.py +29 -0
  135. maxframe/codegen/spe/tests/__init__.py +13 -0
  136. maxframe/codegen/spe/tests/test_remote.py +29 -0
  137. maxframe/codegen/spe/tests/test_spe_codegen.py +141 -0
  138. maxframe/codegen/spe/utils.py +54 -0
  139. maxframe/codegen/tests/__init__.py +13 -0
  140. maxframe/{tests → codegen/tests}/test_codegen.py +3 -5
  141. maxframe/config/__init__.py +1 -1
  142. maxframe/config/config.py +50 -23
  143. maxframe/config/tests/test_config.py +4 -12
  144. maxframe/config/validators.py +5 -0
  145. maxframe/conftest.py +38 -10
  146. maxframe/core/__init__.py +1 -0
  147. maxframe/core/context.py +110 -0
  148. maxframe/core/entity/__init__.py +1 -0
  149. maxframe/core/entity/core.py +0 -7
  150. maxframe/core/entity/objects.py +19 -5
  151. maxframe/core/entity/output_types.py +11 -0
  152. maxframe/core/entity/tests/test_objects.py +11 -12
  153. maxframe/core/entity/tileables.py +3 -1
  154. maxframe/core/entity/utils.py +15 -0
  155. maxframe/core/graph/__init__.py +6 -1
  156. maxframe/core/graph/builder/base.py +5 -1
  157. maxframe/core/graph/core.cpython-38-darwin.so +0 -0
  158. maxframe/core/graph/core.pyx +17 -6
  159. maxframe/core/graph/entity.py +18 -6
  160. maxframe/core/operator/__init__.py +8 -3
  161. maxframe/core/operator/base.py +35 -12
  162. maxframe/core/operator/core.py +37 -14
  163. maxframe/core/operator/fetch.py +5 -18
  164. maxframe/core/operator/objects.py +0 -20
  165. maxframe/core/operator/shuffle.py +6 -72
  166. maxframe/dataframe/__init__.py +1 -0
  167. maxframe/dataframe/accessors/datetime_/core.py +7 -4
  168. maxframe/dataframe/accessors/string_/core.py +9 -6
  169. maxframe/dataframe/arithmetic/core.py +31 -20
  170. maxframe/dataframe/arithmetic/tests/test_arithmetic.py +6 -0
  171. maxframe/dataframe/core.py +98 -91
  172. maxframe/dataframe/datasource/core.py +8 -1
  173. maxframe/dataframe/datasource/date_range.py +8 -0
  174. maxframe/dataframe/datasource/from_index.py +9 -5
  175. maxframe/dataframe/datasource/from_records.py +9 -2
  176. maxframe/dataframe/datasource/from_tensor.py +32 -21
  177. maxframe/dataframe/datasource/read_csv.py +8 -2
  178. maxframe/dataframe/datasource/read_odps_query.py +33 -3
  179. maxframe/dataframe/datasource/read_odps_table.py +20 -5
  180. maxframe/dataframe/datasource/read_parquet.py +8 -3
  181. maxframe/dataframe/datasource/tests/test_datasource.py +33 -0
  182. maxframe/dataframe/datastore/tests/test_to_odps.py +52 -1
  183. maxframe/dataframe/datastore/to_csv.py +7 -3
  184. maxframe/dataframe/datastore/to_odps.py +42 -6
  185. maxframe/dataframe/extensions/__init__.py +6 -1
  186. maxframe/dataframe/extensions/apply_chunk.py +96 -136
  187. maxframe/dataframe/extensions/flatjson.py +3 -2
  188. maxframe/dataframe/extensions/flatmap.py +15 -7
  189. maxframe/dataframe/fetch/core.py +12 -1
  190. maxframe/dataframe/groupby/__init__.py +7 -0
  191. maxframe/dataframe/groupby/aggregation.py +9 -8
  192. maxframe/dataframe/groupby/apply.py +50 -74
  193. maxframe/dataframe/groupby/apply_chunk.py +393 -0
  194. maxframe/dataframe/groupby/core.py +80 -17
  195. maxframe/dataframe/groupby/extensions.py +26 -0
  196. maxframe/dataframe/groupby/fill.py +9 -4
  197. maxframe/dataframe/groupby/sample.py +7 -7
  198. maxframe/dataframe/groupby/tests/test_groupby.py +3 -3
  199. maxframe/dataframe/groupby/transform.py +57 -54
  200. maxframe/dataframe/indexing/align.py +7 -6
  201. maxframe/dataframe/indexing/getitem.py +9 -8
  202. maxframe/dataframe/indexing/iloc.py +28 -23
  203. maxframe/dataframe/indexing/insert.py +7 -3
  204. maxframe/dataframe/indexing/loc.py +9 -8
  205. maxframe/dataframe/indexing/reindex.py +36 -30
  206. maxframe/dataframe/indexing/rename_axis.py +18 -10
  207. maxframe/dataframe/indexing/reset_index.py +0 -2
  208. maxframe/dataframe/indexing/sample.py +13 -9
  209. maxframe/dataframe/indexing/set_axis.py +9 -6
  210. maxframe/dataframe/indexing/setitem.py +8 -5
  211. maxframe/dataframe/indexing/where.py +12 -9
  212. maxframe/dataframe/merge/__init__.py +0 -1
  213. maxframe/dataframe/merge/concat.py +10 -31
  214. maxframe/dataframe/merge/merge.py +2 -24
  215. maxframe/dataframe/misc/__init__.py +6 -0
  216. maxframe/dataframe/misc/_duplicate.py +7 -3
  217. maxframe/dataframe/misc/apply.py +106 -139
  218. maxframe/dataframe/misc/astype.py +3 -2
  219. maxframe/dataframe/misc/case_when.py +11 -7
  220. maxframe/dataframe/misc/cut.py +11 -10
  221. maxframe/dataframe/misc/describe.py +7 -3
  222. maxframe/dataframe/misc/drop.py +13 -11
  223. maxframe/dataframe/misc/eval.py +0 -2
  224. maxframe/dataframe/misc/get_dummies.py +78 -49
  225. maxframe/dataframe/misc/isin.py +13 -10
  226. maxframe/dataframe/misc/map.py +21 -6
  227. maxframe/dataframe/misc/melt.py +8 -1
  228. maxframe/dataframe/misc/pivot.py +232 -0
  229. maxframe/dataframe/misc/pivot_table.py +52 -40
  230. maxframe/dataframe/misc/rechunk.py +59 -0
  231. maxframe/dataframe/misc/shift.py +7 -4
  232. maxframe/dataframe/misc/stack.py +5 -3
  233. maxframe/dataframe/misc/tests/test_misc.py +167 -1
  234. maxframe/dataframe/misc/transform.py +63 -65
  235. maxframe/dataframe/misc/value_counts.py +7 -4
  236. maxframe/dataframe/missing/dropna.py +16 -7
  237. maxframe/dataframe/missing/fillna.py +18 -10
  238. maxframe/dataframe/missing/replace.py +10 -6
  239. maxframe/dataframe/missing/tests/test_missing.py +2 -2
  240. maxframe/dataframe/operators.py +1 -27
  241. maxframe/dataframe/reduction/aggregation.py +65 -3
  242. maxframe/dataframe/reduction/core.py +3 -1
  243. maxframe/dataframe/reduction/median.py +1 -1
  244. maxframe/dataframe/reduction/tests/test_reduction.py +33 -0
  245. maxframe/dataframe/reduction/unique.py +53 -7
  246. maxframe/dataframe/statistics/corr.py +9 -6
  247. maxframe/dataframe/statistics/quantile.py +9 -6
  248. maxframe/dataframe/tseries/to_datetime.py +6 -4
  249. maxframe/dataframe/utils.py +219 -31
  250. maxframe/dataframe/window/rolling.py +7 -4
  251. maxframe/env.py +1 -0
  252. maxframe/errors.py +9 -0
  253. maxframe/extension.py +13 -2
  254. maxframe/io/objects/core.py +67 -51
  255. maxframe/io/objects/tensor.py +73 -17
  256. maxframe/io/objects/tests/test_object_io.py +8 -55
  257. maxframe/io/odpsio/arrow.py +15 -2
  258. maxframe/io/odpsio/schema.py +43 -13
  259. maxframe/io/odpsio/tableio.py +63 -11
  260. maxframe/io/odpsio/tests/test_arrow.py +1 -2
  261. maxframe/io/odpsio/tests/test_schema.py +114 -1
  262. maxframe/io/odpsio/tests/test_tableio.py +42 -0
  263. maxframe/io/odpsio/tests/test_volumeio.py +22 -48
  264. maxframe/learn/__init__.py +2 -2
  265. maxframe/learn/contrib/__init__.py +2 -2
  266. maxframe/learn/contrib/graph/connected_components.py +2 -1
  267. maxframe/learn/contrib/lightgbm/__init__.py +33 -0
  268. maxframe/learn/contrib/lightgbm/_predict.py +138 -0
  269. maxframe/learn/contrib/lightgbm/_train.py +163 -0
  270. maxframe/learn/contrib/lightgbm/callback.py +114 -0
  271. maxframe/learn/contrib/lightgbm/classifier.py +199 -0
  272. maxframe/learn/contrib/lightgbm/core.py +372 -0
  273. maxframe/learn/contrib/lightgbm/dataset.py +153 -0
  274. maxframe/learn/contrib/lightgbm/regressor.py +29 -0
  275. maxframe/learn/contrib/lightgbm/tests/__init__.py +13 -0
  276. maxframe/learn/contrib/lightgbm/tests/test_callback.py +58 -0
  277. maxframe/learn/contrib/models.py +38 -9
  278. maxframe/learn/contrib/utils.py +55 -0
  279. maxframe/learn/contrib/xgboost/callback.py +86 -0
  280. maxframe/learn/contrib/xgboost/classifier.py +26 -30
  281. maxframe/learn/contrib/xgboost/core.py +53 -42
  282. maxframe/learn/contrib/xgboost/dmatrix.py +19 -12
  283. maxframe/learn/contrib/xgboost/predict.py +16 -9
  284. maxframe/learn/contrib/xgboost/regressor.py +28 -27
  285. maxframe/learn/contrib/xgboost/tests/test_callback.py +41 -0
  286. maxframe/learn/contrib/xgboost/train.py +59 -16
  287. maxframe/learn/core.py +252 -0
  288. maxframe/learn/datasets/__init__.py +20 -0
  289. maxframe/learn/datasets/samples_generator.py +628 -0
  290. maxframe/learn/linear_model/__init__.py +15 -0
  291. maxframe/learn/linear_model/_base.py +163 -0
  292. maxframe/learn/linear_model/_lin_reg.py +175 -0
  293. maxframe/learn/metrics/__init__.py +25 -0
  294. maxframe/learn/metrics/_check_targets.py +95 -0
  295. maxframe/learn/metrics/_classification.py +1121 -0
  296. maxframe/learn/metrics/_regression.py +256 -0
  297. maxframe/learn/model_selection/__init__.py +15 -0
  298. maxframe/learn/model_selection/_split.py +451 -0
  299. maxframe/learn/model_selection/tests/__init__.py +13 -0
  300. maxframe/learn/model_selection/tests/test_split.py +156 -0
  301. maxframe/learn/preprocessing/__init__.py +16 -0
  302. maxframe/learn/preprocessing/_data/__init__.py +17 -0
  303. maxframe/learn/preprocessing/_data/min_max_scaler.py +390 -0
  304. maxframe/learn/preprocessing/_data/normalize.py +127 -0
  305. maxframe/learn/preprocessing/_data/standard_scaler.py +503 -0
  306. maxframe/learn/preprocessing/_data/utils.py +79 -0
  307. maxframe/learn/preprocessing/_label/__init__.py +16 -0
  308. maxframe/learn/preprocessing/_label/_label_binarizer.py +599 -0
  309. maxframe/learn/preprocessing/_label/_label_encoder.py +174 -0
  310. maxframe/learn/utils/__init__.py +4 -0
  311. maxframe/learn/utils/_encode.py +314 -0
  312. maxframe/learn/utils/checks.py +161 -0
  313. maxframe/learn/utils/core.py +33 -0
  314. maxframe/learn/utils/extmath.py +176 -0
  315. maxframe/learn/utils/multiclass.py +292 -0
  316. maxframe/learn/utils/shuffle.py +114 -0
  317. maxframe/learn/utils/sparsefuncs.py +87 -0
  318. maxframe/learn/utils/validation.py +775 -0
  319. maxframe/lib/__init__.py +0 -2
  320. maxframe/lib/compat.py +145 -0
  321. maxframe/lib/filesystem/_oss_lib/glob.py +1 -1
  322. maxframe/lib/mmh3.cpython-38-darwin.so +0 -0
  323. maxframe/lib/sparse/__init__.py +10 -15
  324. maxframe/lib/sparse/array.py +45 -33
  325. maxframe/lib/sparse/core.py +0 -2
  326. maxframe/lib/sparse/linalg.py +31 -0
  327. maxframe/lib/sparse/matrix.py +5 -2
  328. maxframe/lib/sparse/tests/__init__.py +0 -2
  329. maxframe/lib/sparse/tests/test_sparse.py +53 -53
  330. maxframe/lib/sparse/vector.py +0 -2
  331. maxframe/mixin.py +59 -2
  332. maxframe/opcodes.py +13 -5
  333. maxframe/protocol.py +67 -14
  334. maxframe/remote/core.py +16 -14
  335. maxframe/remote/run_script.py +6 -3
  336. maxframe/serialization/__init__.py +2 -0
  337. maxframe/serialization/core.cpython-38-darwin.so +0 -0
  338. maxframe/serialization/core.pxd +3 -0
  339. maxframe/serialization/core.pyi +3 -1
  340. maxframe/serialization/core.pyx +82 -4
  341. maxframe/serialization/pandas.py +5 -1
  342. maxframe/serialization/serializables/core.py +6 -5
  343. maxframe/serialization/serializables/field.py +2 -2
  344. maxframe/serialization/serializables/tests/test_field_type.py +3 -5
  345. maxframe/serialization/tests/test_serial.py +27 -0
  346. maxframe/session.py +4 -71
  347. maxframe/sperunner.py +165 -0
  348. maxframe/tensor/__init__.py +35 -2
  349. maxframe/tensor/arithmetic/__init__.py +2 -4
  350. maxframe/tensor/arithmetic/abs.py +0 -2
  351. maxframe/tensor/arithmetic/absolute.py +0 -2
  352. maxframe/tensor/arithmetic/add.py +34 -4
  353. maxframe/tensor/arithmetic/angle.py +0 -2
  354. maxframe/tensor/arithmetic/arccos.py +1 -4
  355. maxframe/tensor/arithmetic/arccosh.py +1 -3
  356. maxframe/tensor/arithmetic/arcsin.py +0 -2
  357. maxframe/tensor/arithmetic/arcsinh.py +0 -2
  358. maxframe/tensor/arithmetic/arctan.py +0 -2
  359. maxframe/tensor/arithmetic/arctan2.py +0 -2
  360. maxframe/tensor/arithmetic/arctanh.py +0 -2
  361. maxframe/tensor/arithmetic/around.py +0 -2
  362. maxframe/tensor/arithmetic/bitand.py +0 -2
  363. maxframe/tensor/arithmetic/bitor.py +1 -3
  364. maxframe/tensor/arithmetic/bitxor.py +1 -3
  365. maxframe/tensor/arithmetic/cbrt.py +0 -2
  366. maxframe/tensor/arithmetic/ceil.py +0 -2
  367. maxframe/tensor/arithmetic/clip.py +13 -13
  368. maxframe/tensor/arithmetic/conj.py +0 -2
  369. maxframe/tensor/arithmetic/copysign.py +0 -2
  370. maxframe/tensor/arithmetic/core.py +47 -39
  371. maxframe/tensor/arithmetic/cos.py +1 -3
  372. maxframe/tensor/arithmetic/cosh.py +0 -2
  373. maxframe/tensor/arithmetic/deg2rad.py +0 -2
  374. maxframe/tensor/arithmetic/degrees.py +0 -2
  375. maxframe/tensor/arithmetic/divide.py +0 -2
  376. maxframe/tensor/arithmetic/equal.py +0 -2
  377. maxframe/tensor/arithmetic/exp.py +1 -3
  378. maxframe/tensor/arithmetic/exp2.py +0 -2
  379. maxframe/tensor/arithmetic/expm1.py +0 -2
  380. maxframe/tensor/arithmetic/fabs.py +0 -2
  381. maxframe/tensor/arithmetic/fix.py +0 -2
  382. maxframe/tensor/arithmetic/float_power.py +0 -2
  383. maxframe/tensor/arithmetic/floor.py +0 -2
  384. maxframe/tensor/arithmetic/floordiv.py +0 -2
  385. maxframe/tensor/arithmetic/fmax.py +0 -2
  386. maxframe/tensor/arithmetic/fmin.py +0 -2
  387. maxframe/tensor/arithmetic/fmod.py +0 -2
  388. maxframe/tensor/arithmetic/frexp.py +6 -2
  389. maxframe/tensor/arithmetic/greater.py +0 -2
  390. maxframe/tensor/arithmetic/greater_equal.py +0 -2
  391. maxframe/tensor/arithmetic/hypot.py +0 -2
  392. maxframe/tensor/arithmetic/i0.py +1 -3
  393. maxframe/tensor/arithmetic/imag.py +0 -2
  394. maxframe/tensor/arithmetic/invert.py +1 -3
  395. maxframe/tensor/arithmetic/isclose.py +0 -2
  396. maxframe/tensor/arithmetic/iscomplex.py +0 -2
  397. maxframe/tensor/arithmetic/isfinite.py +1 -3
  398. maxframe/tensor/arithmetic/isinf.py +0 -2
  399. maxframe/tensor/arithmetic/isnan.py +0 -2
  400. maxframe/tensor/arithmetic/isreal.py +0 -2
  401. maxframe/tensor/arithmetic/ldexp.py +0 -2
  402. maxframe/tensor/arithmetic/less.py +0 -2
  403. maxframe/tensor/arithmetic/less_equal.py +0 -2
  404. maxframe/tensor/arithmetic/log.py +1 -3
  405. maxframe/tensor/arithmetic/log10.py +1 -3
  406. maxframe/tensor/arithmetic/log1p.py +1 -3
  407. maxframe/tensor/arithmetic/log2.py +1 -3
  408. maxframe/tensor/arithmetic/logaddexp.py +0 -2
  409. maxframe/tensor/arithmetic/logaddexp2.py +0 -2
  410. maxframe/tensor/arithmetic/logical_and.py +0 -2
  411. maxframe/tensor/arithmetic/logical_not.py +1 -3
  412. maxframe/tensor/arithmetic/logical_or.py +0 -2
  413. maxframe/tensor/arithmetic/logical_xor.py +0 -2
  414. maxframe/tensor/arithmetic/lshift.py +0 -2
  415. maxframe/tensor/arithmetic/maximum.py +0 -2
  416. maxframe/tensor/arithmetic/minimum.py +0 -2
  417. maxframe/tensor/arithmetic/mod.py +0 -2
  418. maxframe/tensor/arithmetic/modf.py +6 -2
  419. maxframe/tensor/arithmetic/multiply.py +37 -4
  420. maxframe/tensor/arithmetic/nan_to_num.py +0 -2
  421. maxframe/tensor/arithmetic/negative.py +0 -2
  422. maxframe/tensor/arithmetic/nextafter.py +0 -2
  423. maxframe/tensor/arithmetic/not_equal.py +0 -2
  424. maxframe/tensor/arithmetic/positive.py +0 -2
  425. maxframe/tensor/arithmetic/power.py +0 -2
  426. maxframe/tensor/arithmetic/rad2deg.py +0 -2
  427. maxframe/tensor/arithmetic/radians.py +0 -2
  428. maxframe/tensor/arithmetic/real.py +0 -2
  429. maxframe/tensor/arithmetic/reciprocal.py +5 -3
  430. maxframe/tensor/arithmetic/rint.py +1 -3
  431. maxframe/tensor/arithmetic/rshift.py +0 -2
  432. maxframe/tensor/arithmetic/setimag.py +0 -2
  433. maxframe/tensor/arithmetic/setreal.py +0 -2
  434. maxframe/tensor/arithmetic/sign.py +0 -2
  435. maxframe/tensor/arithmetic/signbit.py +0 -2
  436. maxframe/tensor/arithmetic/sin.py +0 -2
  437. maxframe/tensor/arithmetic/sinc.py +1 -3
  438. maxframe/tensor/arithmetic/sinh.py +0 -2
  439. maxframe/tensor/arithmetic/spacing.py +0 -2
  440. maxframe/tensor/arithmetic/sqrt.py +0 -2
  441. maxframe/tensor/arithmetic/square.py +0 -2
  442. maxframe/tensor/arithmetic/subtract.py +4 -2
  443. maxframe/tensor/arithmetic/tan.py +0 -2
  444. maxframe/tensor/arithmetic/tanh.py +0 -2
  445. maxframe/tensor/arithmetic/tests/__init__.py +0 -2
  446. maxframe/tensor/arithmetic/tests/test_arithmetic.py +43 -9
  447. maxframe/tensor/arithmetic/truediv.py +0 -2
  448. maxframe/tensor/arithmetic/trunc.py +0 -2
  449. maxframe/tensor/arithmetic/utils.py +32 -6
  450. maxframe/tensor/array_utils.py +3 -25
  451. maxframe/tensor/core.py +6 -6
  452. maxframe/tensor/datasource/__init__.py +10 -2
  453. maxframe/tensor/datasource/arange.py +0 -2
  454. maxframe/tensor/datasource/array.py +3 -22
  455. maxframe/tensor/datasource/core.py +15 -10
  456. maxframe/tensor/datasource/diag.py +140 -0
  457. maxframe/tensor/datasource/diagflat.py +69 -0
  458. maxframe/tensor/datasource/empty.py +0 -2
  459. maxframe/tensor/datasource/eye.py +95 -0
  460. maxframe/tensor/datasource/from_dataframe.py +0 -2
  461. maxframe/tensor/datasource/from_dense.py +0 -17
  462. maxframe/tensor/datasource/from_sparse.py +0 -2
  463. maxframe/tensor/datasource/full.py +0 -2
  464. maxframe/tensor/datasource/identity.py +54 -0
  465. maxframe/tensor/datasource/indices.py +115 -0
  466. maxframe/tensor/datasource/linspace.py +140 -0
  467. maxframe/tensor/datasource/meshgrid.py +135 -0
  468. maxframe/tensor/datasource/ones.py +8 -3
  469. maxframe/tensor/datasource/tests/test_datasource.py +32 -1
  470. maxframe/tensor/datasource/tri_array.py +107 -0
  471. maxframe/tensor/datasource/zeros.py +7 -3
  472. maxframe/tensor/extensions/__init__.py +31 -0
  473. maxframe/tensor/extensions/accessor.py +25 -0
  474. maxframe/tensor/extensions/apply_chunk.py +137 -0
  475. maxframe/tensor/indexing/__init__.py +1 -1
  476. maxframe/tensor/indexing/choose.py +8 -6
  477. maxframe/tensor/indexing/compress.py +0 -2
  478. maxframe/tensor/indexing/extract.py +0 -2
  479. maxframe/tensor/indexing/fill_diagonal.py +9 -6
  480. maxframe/tensor/indexing/flatnonzero.py +1 -3
  481. maxframe/tensor/indexing/getitem.py +10 -43
  482. maxframe/tensor/indexing/nonzero.py +2 -4
  483. maxframe/tensor/indexing/setitem.py +19 -9
  484. maxframe/tensor/indexing/slice.py +6 -3
  485. maxframe/tensor/indexing/take.py +0 -2
  486. maxframe/tensor/indexing/tests/__init__.py +0 -2
  487. maxframe/tensor/indexing/tests/test_indexing.py +0 -2
  488. maxframe/tensor/indexing/unravel_index.py +6 -6
  489. maxframe/tensor/lib/__init__.py +16 -0
  490. maxframe/tensor/lib/index_tricks.py +404 -0
  491. maxframe/tensor/linalg/__init__.py +36 -0
  492. maxframe/tensor/linalg/dot.py +145 -0
  493. maxframe/tensor/linalg/inner.py +36 -0
  494. maxframe/tensor/linalg/inv.py +83 -0
  495. maxframe/tensor/linalg/lu.py +115 -0
  496. maxframe/tensor/linalg/matmul.py +225 -0
  497. maxframe/tensor/linalg/qr.py +124 -0
  498. maxframe/tensor/linalg/solve_triangular.py +103 -0
  499. maxframe/tensor/linalg/svd.py +167 -0
  500. maxframe/tensor/linalg/tensordot.py +213 -0
  501. maxframe/tensor/linalg/vdot.py +73 -0
  502. maxframe/tensor/merge/__init__.py +4 -0
  503. maxframe/tensor/merge/append.py +74 -0
  504. maxframe/tensor/merge/column_stack.py +63 -0
  505. maxframe/tensor/merge/concatenate.py +3 -2
  506. maxframe/tensor/merge/dstack.py +71 -0
  507. maxframe/tensor/merge/hstack.py +70 -0
  508. maxframe/tensor/merge/stack.py +0 -2
  509. maxframe/tensor/merge/tests/test_merge.py +0 -2
  510. maxframe/tensor/misc/__init__.py +18 -5
  511. maxframe/tensor/misc/astype.py +10 -8
  512. maxframe/tensor/misc/broadcast_to.py +1 -1
  513. maxframe/tensor/misc/copy.py +64 -0
  514. maxframe/tensor/misc/diff.py +115 -0
  515. maxframe/tensor/misc/flatten.py +63 -0
  516. maxframe/tensor/misc/in1d.py +94 -0
  517. maxframe/tensor/misc/isin.py +130 -0
  518. maxframe/tensor/misc/ndim.py +53 -0
  519. maxframe/tensor/misc/ravel.py +0 -2
  520. maxframe/tensor/misc/repeat.py +129 -0
  521. maxframe/tensor/misc/searchsorted.py +147 -0
  522. maxframe/tensor/misc/setdiff1d.py +58 -0
  523. maxframe/tensor/misc/squeeze.py +117 -0
  524. maxframe/tensor/misc/swapaxes.py +113 -0
  525. maxframe/tensor/misc/tests/test_misc.py +0 -2
  526. maxframe/tensor/misc/transpose.py +8 -4
  527. maxframe/tensor/misc/trapezoid.py +123 -0
  528. maxframe/tensor/misc/unique.py +0 -1
  529. maxframe/tensor/misc/where.py +10 -8
  530. maxframe/tensor/operators.py +0 -34
  531. maxframe/tensor/random/__init__.py +3 -5
  532. maxframe/tensor/random/binomial.py +0 -2
  533. maxframe/tensor/random/bytes.py +0 -2
  534. maxframe/tensor/random/chisquare.py +0 -2
  535. maxframe/tensor/random/choice.py +9 -8
  536. maxframe/tensor/random/core.py +20 -5
  537. maxframe/tensor/random/dirichlet.py +0 -2
  538. maxframe/tensor/random/exponential.py +0 -2
  539. maxframe/tensor/random/f.py +2 -4
  540. maxframe/tensor/random/gamma.py +0 -2
  541. maxframe/tensor/random/geometric.py +0 -2
  542. maxframe/tensor/random/gumbel.py +0 -2
  543. maxframe/tensor/random/hypergeometric.py +0 -2
  544. maxframe/tensor/random/laplace.py +2 -4
  545. maxframe/tensor/random/logistic.py +0 -2
  546. maxframe/tensor/random/lognormal.py +0 -2
  547. maxframe/tensor/random/logseries.py +0 -2
  548. maxframe/tensor/random/multinomial.py +0 -2
  549. maxframe/tensor/random/multivariate_normal.py +0 -2
  550. maxframe/tensor/random/negative_binomial.py +0 -2
  551. maxframe/tensor/random/noncentral_chisquare.py +0 -2
  552. maxframe/tensor/random/noncentral_f.py +1 -3
  553. maxframe/tensor/random/normal.py +0 -2
  554. maxframe/tensor/random/pareto.py +0 -2
  555. maxframe/tensor/random/permutation.py +6 -3
  556. maxframe/tensor/random/poisson.py +0 -2
  557. maxframe/tensor/random/power.py +0 -2
  558. maxframe/tensor/random/rand.py +0 -2
  559. maxframe/tensor/random/randint.py +0 -2
  560. maxframe/tensor/random/randn.py +0 -2
  561. maxframe/tensor/random/random_integers.py +0 -2
  562. maxframe/tensor/random/random_sample.py +0 -2
  563. maxframe/tensor/random/rayleigh.py +0 -2
  564. maxframe/tensor/random/standard_cauchy.py +0 -2
  565. maxframe/tensor/random/standard_exponential.py +0 -2
  566. maxframe/tensor/random/standard_gamma.py +0 -2
  567. maxframe/tensor/random/standard_normal.py +0 -2
  568. maxframe/tensor/random/standard_t.py +0 -2
  569. maxframe/tensor/random/tests/__init__.py +0 -2
  570. maxframe/tensor/random/tests/test_random.py +0 -2
  571. maxframe/tensor/random/triangular.py +0 -2
  572. maxframe/tensor/random/uniform.py +0 -2
  573. maxframe/tensor/random/vonmises.py +0 -2
  574. maxframe/tensor/random/wald.py +0 -2
  575. maxframe/tensor/random/weibull.py +0 -2
  576. maxframe/tensor/random/zipf.py +0 -2
  577. maxframe/tensor/reduction/__init__.py +0 -2
  578. maxframe/tensor/reduction/all.py +0 -2
  579. maxframe/tensor/reduction/allclose.py +0 -2
  580. maxframe/tensor/reduction/any.py +0 -2
  581. maxframe/tensor/reduction/argmax.py +1 -3
  582. maxframe/tensor/reduction/argmin.py +1 -3
  583. maxframe/tensor/reduction/array_equal.py +0 -2
  584. maxframe/tensor/reduction/core.py +0 -2
  585. maxframe/tensor/reduction/count_nonzero.py +0 -2
  586. maxframe/tensor/reduction/cumprod.py +0 -2
  587. maxframe/tensor/reduction/cumsum.py +0 -2
  588. maxframe/tensor/reduction/max.py +0 -2
  589. maxframe/tensor/reduction/mean.py +0 -2
  590. maxframe/tensor/reduction/min.py +0 -2
  591. maxframe/tensor/reduction/nanargmax.py +0 -2
  592. maxframe/tensor/reduction/nanargmin.py +0 -2
  593. maxframe/tensor/reduction/nancumprod.py +0 -2
  594. maxframe/tensor/reduction/nancumsum.py +0 -2
  595. maxframe/tensor/reduction/nanmax.py +0 -2
  596. maxframe/tensor/reduction/nanmean.py +0 -2
  597. maxframe/tensor/reduction/nanmin.py +0 -2
  598. maxframe/tensor/reduction/nanprod.py +0 -2
  599. maxframe/tensor/reduction/nanstd.py +0 -2
  600. maxframe/tensor/reduction/nansum.py +0 -2
  601. maxframe/tensor/reduction/nanvar.py +0 -2
  602. maxframe/tensor/reduction/prod.py +0 -2
  603. maxframe/tensor/reduction/std.py +0 -2
  604. maxframe/tensor/reduction/sum.py +0 -2
  605. maxframe/tensor/reduction/tests/test_reduction.py +1 -4
  606. maxframe/tensor/reduction/var.py +0 -2
  607. maxframe/tensor/reshape/__init__.py +0 -2
  608. maxframe/tensor/reshape/reshape.py +6 -5
  609. maxframe/tensor/reshape/tests/__init__.py +0 -2
  610. maxframe/tensor/reshape/tests/test_reshape.py +0 -2
  611. maxframe/tensor/sort/__init__.py +16 -0
  612. maxframe/tensor/sort/argsort.py +150 -0
  613. maxframe/tensor/sort/sort.py +295 -0
  614. maxframe/tensor/special/__init__.py +37 -0
  615. maxframe/tensor/special/core.py +38 -0
  616. maxframe/tensor/special/misc.py +142 -0
  617. maxframe/tensor/special/statistical.py +56 -0
  618. maxframe/tensor/statistics/__init__.py +5 -0
  619. maxframe/tensor/statistics/average.py +143 -0
  620. maxframe/tensor/statistics/bincount.py +133 -0
  621. maxframe/tensor/statistics/quantile.py +10 -8
  622. maxframe/tensor/ufunc/__init__.py +0 -2
  623. maxframe/tensor/ufunc/ufunc.py +0 -2
  624. maxframe/tensor/utils.py +21 -3
  625. maxframe/tests/test_protocol.py +3 -3
  626. maxframe/tests/test_utils.py +210 -1
  627. maxframe/tests/utils.py +67 -1
  628. maxframe/udf.py +76 -6
  629. maxframe/utils.py +418 -17
  630. {maxframe-1.3.1.dist-info → maxframe-2.0.0b1.dist-info}/METADATA +4 -1
  631. maxframe-2.0.0b1.dist-info/RECORD +939 -0
  632. maxframe_client/clients/framedriver.py +19 -3
  633. maxframe_client/fetcher.py +113 -6
  634. maxframe_client/session/odps.py +173 -38
  635. maxframe_client/session/task.py +3 -1
  636. maxframe_client/tests/test_session.py +41 -5
  637. maxframe-1.3.1.dist-info/RECORD +0 -705
  638. {maxframe-1.3.1.dist-info → maxframe-2.0.0b1.dist-info}/WHEEL +0 -0
  639. {maxframe-1.3.1.dist-info → maxframe-2.0.0b1.dist-info}/top_level.txt +0 -0
Binary file
maxframe/_utils.pyi ADDED
@@ -0,0 +1,21 @@
1
+ # Copyright 1999-2025 Alibaba Group Holding Ltd.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ from typing import Callable, Optional, Type
16
+
17
+ def to_str(s, encoding: Optional[str] = None) -> str: ...
18
+ def to_binary(s, encoding: Optional[str] = None) -> bytes: ...
19
+ def register_tokenizer(cls: Type, handler: Callable) -> None: ...
20
+ def reset_id_random_seed() -> None: ...
21
+ def new_random_id(byte_len: int) -> bytes: ...
maxframe/_utils.pyx CHANGED
@@ -22,7 +22,6 @@ import pickle
22
22
  import pkgutil
23
23
  import time
24
24
  import types
25
- import uuid
26
25
  import warnings
27
26
  from datetime import date, datetime, timedelta, tzinfo
28
27
  from enum import Enum
@@ -283,8 +282,8 @@ cdef inline tuple tokenize_numpy(ob):
283
282
  try:
284
283
  data = mmh_hash_bytes(pickle.dumps(ob, pickle.HIGHEST_PROTOCOL))
285
284
  except:
286
- # nothing can do, generate uuid
287
- data = uuid.uuid4().hex
285
+ # nothing can do, generate random id
286
+ data = new_random_id(32).hex().encode()
288
287
  else:
289
288
  try:
290
289
  data = mmh_hash_bytes(ob.ravel().view('i1').data)
@@ -397,12 +396,14 @@ def tokenize_pickled_with_cache(ob):
397
396
 
398
397
  def tokenize_cupy(ob):
399
398
  from .serialization import serialize
399
+
400
400
  header, _buffers = serialize(ob)
401
401
  return iterative_tokenize([header, ob.data.ptr])
402
402
 
403
403
 
404
404
  def tokenize_cudf(ob):
405
405
  from .serialization import serialize
406
+
406
407
  header, buffers = serialize(ob)
407
408
  return iterative_tokenize([header] + [(buf.ptr, buf.size) for buf in buffers])
408
409
 
@@ -0,0 +1,27 @@
1
+ # Copyright 1999-2025 Alibaba Group Holding Ltd.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ from .core import (
16
+ BUILTIN_ENGINE_DPE,
17
+ BUILTIN_ENGINE_MCSQL,
18
+ BUILTIN_ENGINE_SPE,
19
+ AbstractUDF,
20
+ CodeGenResult,
21
+ DAGCodeContext,
22
+ DAGCodeGenerator,
23
+ DagOperatorAdapter,
24
+ EngineAcceptance,
25
+ UserCodeMixin,
26
+ register_engine_codegen,
27
+ )
@@ -23,17 +23,17 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union
23
23
  from odps.types import OdpsSchema
24
24
  from odps.utils import camel_to_underline
25
25
 
26
- from .core import OperatorType, Tileable, TileableGraph
27
- from .core.operator import Fetch, Operator
28
- from .extension import iter_extensions
29
- from .io.odpsio import build_dataframe_table_meta
30
- from .io.odpsio.schema import pandas_to_odps_schema
31
- from .lib import wrapped_pickle as pickle
32
- from .protocol import DataFrameTableMeta, ResultInfo
33
- from .serialization import PickleContainer
34
- from .serialization.serializables import Serializable, StringField
35
- from .typing_ import PandasObjectTypes
36
- from .udf import MarkedFunction, PythonPackOptions
26
+ from ..core import OperatorType, Tileable, TileableGraph
27
+ from ..core.operator import Fetch, Operator
28
+ from ..extension import iter_extensions
29
+ from ..io.odpsio import build_dataframe_table_meta
30
+ from ..io.odpsio.schema import pandas_to_odps_schema
31
+ from ..lib import wrapped_pickle as pickle
32
+ from ..protocol import DataFrameTableMeta, ResultInfo
33
+ from ..serialization import PickleContainer
34
+ from ..serialization.serializables import Serializable, StringField
35
+ from ..typing_ import PandasObjectTypes
36
+ from ..udf import MarkedFunction, PythonPackOptions
37
37
 
38
38
  if TYPE_CHECKING:
39
39
  from odpsctx import ODPSSessionContext
@@ -129,7 +129,8 @@ class UserCodeMixin:
129
129
  return f"{{{', '.join(items)}}}"
130
130
 
131
131
  if isinstance(obj, tuple):
132
- return f"({', '.join([cls.obj_to_python_expr(sub_obj) for sub_obj in obj])}{',' if len(obj) == 1 else ''})"
132
+ obj_exprs = [cls.obj_to_python_expr(sub_obj) for sub_obj in obj]
133
+ return f"({', '.join(obj_exprs)}{',' if len(obj) == 1 else ''})"
133
134
 
134
135
  if isinstance(obj, set):
135
136
  return (
@@ -162,10 +163,14 @@ class UserCodeMixin:
162
163
  Returns
163
164
  -------
164
165
  str :
165
- The code snippets of pickling, the final variable is called 'pickled_data' by default.
166
+ The code snippets of pickling, the final variable is called
167
+ 'pickled_data' by default.
166
168
  """
167
169
  pickled, buffers = cls.dump_pickled_data(code_to_pickle)
168
- pickle_loads_expr = f"cloudpickle.loads({cls.obj_to_python_expr(pickled)}, buffers={cls.obj_to_python_expr(buffers)})"
170
+ pickle_loads_expr = (
171
+ f"cloudpickle.loads({cls.obj_to_python_expr(pickled)}, "
172
+ f"buffers={cls.obj_to_python_expr(buffers)})"
173
+ )
169
174
  if unpicked_data_var_name:
170
175
  return f"{unpicked_data_var_name} = {pickle_loads_expr}"
171
176
 
@@ -187,7 +192,7 @@ class UserCodeMixin:
187
192
  return pickled, buffers
188
193
 
189
194
 
190
- class BigDagCodeContext(metaclass=abc.ABCMeta):
195
+ class DAGCodeContext(metaclass=abc.ABCMeta):
191
196
  def __init__(self, session_id: str = None, subdag_id: str = None):
192
197
  self._session_id = session_id
193
198
  self._subdag_id = subdag_id
@@ -204,6 +209,10 @@ class BigDagCodeContext(metaclass=abc.ABCMeta):
204
209
  def session_id(self) -> str:
205
210
  return self._session_id
206
211
 
212
+ @property
213
+ def subdag_id(self) -> str:
214
+ return self._subdag_id
215
+
207
216
  def register_udf(self, udf: AbstractUDF):
208
217
  udf.session_id = self._session_id
209
218
  self._udfs[udf.name] = udf
@@ -293,7 +302,10 @@ class EngineAcceptance(Enum):
293
302
  """
294
303
  DENY: The operator is not accepted by the current engine.
295
304
  ACCEPT: The operator is accepted by the current engine, and doesn't break from here.
296
- BREAK: The operator is accepted by the current engine, but should break from here.
305
+ BREAK_BEFORE: The operator is accepted by the current engine, but should break before
306
+ its execution.
307
+ BREAK_AFTER: The operator is accepted by the current engine, but should break after
308
+ its execution.
297
309
  PREDECESSOR: The acceptance of the operator is decided by engines of its
298
310
  predecessors. If acceptance of all predecessors are SUCCESSOR, the acceptance
299
311
  of current operator is SUCCESSOR. Otherwise the engine selected in predecessors
@@ -305,9 +317,10 @@ class EngineAcceptance(Enum):
305
317
 
306
318
  DENY = 0
307
319
  ACCEPT = 1
308
- BREAK = 2
320
+ BREAK_AFTER = 2
309
321
  PREDECESSOR = 3
310
322
  SUCCESSOR = 4
323
+ BREAK_BEFORE = 5
311
324
 
312
325
  @classmethod
313
326
  def _missing_(cls, pred: bool) -> "EngineAcceptance":
@@ -327,18 +340,16 @@ class EngineAcceptance(Enum):
327
340
  return cls.ACCEPT if pred else cls.DENY
328
341
 
329
342
 
330
- class BigDagOperatorAdapter(metaclass=abc.ABCMeta):
343
+ class DagOperatorAdapter(metaclass=abc.ABCMeta):
331
344
  # todo handle refcount issue when generated code is being executed
332
345
  def accepts(self, op: Operator) -> EngineAcceptance:
333
346
  return EngineAcceptance.ACCEPT
334
347
 
335
348
  @abc.abstractmethod
336
- def generate_code(self, op: OperatorType, context: BigDagCodeContext) -> List[str]:
349
+ def generate_code(self, op: Operator, context: DAGCodeContext) -> List[str]:
337
350
  raise NotImplementedError
338
351
 
339
- def generate_comment(
340
- self, op: OperatorType, context: BigDagCodeContext
341
- ) -> List[str]:
352
+ def generate_comment(self, op: Operator, context: DAGCodeContext) -> List[str]:
342
353
  """
343
354
  Generate the comment codes before actual ones.
344
355
 
@@ -346,8 +357,8 @@ class BigDagOperatorAdapter(metaclass=abc.ABCMeta):
346
357
  ----------
347
358
  op : Operator
348
359
  The operator instance.
349
- context : BigDagCodeContext
350
- The BigDagCodeContext instance.
360
+ context : DAGCodeContext
361
+ The DagCodeContext instance.
351
362
 
352
363
  Returns
353
364
  -------
@@ -356,9 +367,7 @@ class BigDagOperatorAdapter(metaclass=abc.ABCMeta):
356
367
  """
357
368
  return list()
358
369
 
359
- def generate_pre_op_code(
360
- self, op: Operator, context: BigDagCodeContext
361
- ) -> List[str]:
370
+ def generate_pre_op_code(self, op: Operator, context: DAGCodeContext) -> List[str]:
362
371
  """
363
372
  Generate the codes before actually handling the operator.
364
373
  This method is usually implemented in the base class of each engine.
@@ -367,8 +376,8 @@ class BigDagOperatorAdapter(metaclass=abc.ABCMeta):
367
376
  ----------
368
377
  op : Operator
369
378
  The operator instance.
370
- context : BigDagCodeContext
371
- The BigDagCodeContext instance.
379
+ context : DAGCodeContext
380
+ The DagCodeContext instance.
372
381
 
373
382
  Returns
374
383
  -------
@@ -377,9 +386,7 @@ class BigDagOperatorAdapter(metaclass=abc.ABCMeta):
377
386
  """
378
387
  return list()
379
388
 
380
- def generate_post_op_code(
381
- self, op: Operator, context: BigDagCodeContext
382
- ) -> List[str]:
389
+ def generate_post_op_code(self, op: Operator, context: DAGCodeContext) -> List[str]:
383
390
  """
384
391
  Generate the codes after actually handling the operator.
385
392
  This method is usually implemented in the base class of each engine.
@@ -388,8 +395,8 @@ class BigDagOperatorAdapter(metaclass=abc.ABCMeta):
388
395
  ----------
389
396
  op : Operator
390
397
  The operator instance.
391
- context : BigDagCodeContext
392
- The BigDagCodeContext instance.
398
+ context : DAGCodeContext
399
+ The DagCodeContext instance.
393
400
 
394
401
  Returns
395
402
  -------
@@ -399,20 +406,21 @@ class BigDagOperatorAdapter(metaclass=abc.ABCMeta):
399
406
  return list()
400
407
 
401
408
 
402
- _engine_to_codegen: Dict[str, Type["BigDagCodeGenerator"]] = dict()
409
+ _engine_to_codegen: Dict[str, Type["DAGCodeGenerator"]] = dict()
403
410
 
404
411
 
405
- def register_engine_codegen(type_: Type["BigDagCodeGenerator"]):
412
+ def register_engine_codegen(type_: Type["DAGCodeGenerator"]):
406
413
  _engine_to_codegen[type_.engine_type] = type_
407
414
  return type_
408
415
 
409
416
 
417
+ BUILTIN_ENGINE_DPE = "DPE"
410
418
  BUILTIN_ENGINE_SPE = "SPE"
411
419
  BUILTIN_ENGINE_MCSQL = "MCSQL"
412
420
 
413
421
 
414
- class BigDagCodeGenerator(metaclass=abc.ABCMeta):
415
- _context: BigDagCodeContext
422
+ class DAGCodeGenerator(metaclass=abc.ABCMeta):
423
+ _context: DAGCodeContext
416
424
 
417
425
  engine_type: Optional[str] = None
418
426
  engine_priority: int = 0
@@ -442,18 +450,16 @@ class BigDagCodeGenerator(metaclass=abc.ABCMeta):
442
450
  return [e.engine_type for e in engines]
443
451
 
444
452
  @classmethod
445
- def get_by_engine_type(cls, engine_type: str) -> Type["BigDagCodeGenerator"]:
453
+ def get_by_engine_type(cls, engine_type: str) -> Type["DAGCodeGenerator"]:
446
454
  cls._load_engine_extensions()
447
455
  return _engine_to_codegen[engine_type]
448
456
 
449
457
  @abc.abstractmethod
450
- def get_op_adapter(
451
- self, op_type: Type[OperatorType]
452
- ) -> Type[BigDagOperatorAdapter]:
458
+ def get_op_adapter(self, op_type: Type[OperatorType]) -> Type[DagOperatorAdapter]:
453
459
  raise NotImplementedError
454
460
 
455
461
  @abc.abstractmethod
456
- def _init_context(self, session_id: str, subdag_id: str) -> BigDagCodeContext:
462
+ def _init_context(self, session_id: str, subdag_id: str) -> DAGCodeContext:
457
463
  raise NotImplementedError
458
464
 
459
465
  def _generate_delete_code(self, var_name: str) -> List[str]:
@@ -0,0 +1,16 @@
1
+ # Copyright 1999-2025 Alibaba Group Holding Ltd.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ from . import dataframe, learn, objects, remote, tensor
16
+ from .core import SPECodeGenerator
@@ -0,0 +1,307 @@
1
+ # Copyright 1999-2025 Alibaba Group Holding Ltd.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import itertools
16
+ from abc import ABC, abstractmethod
17
+ from collections import defaultdict
18
+ from typing import Any, Callable, Dict, Iterable, List, Optional, Type, Union
19
+
20
+ import pandas as pd
21
+ from numpy import dtype as npdtype
22
+ from numpy import ndarray
23
+ from pandas.tseries.offsets import BaseOffset
24
+
25
+ from ...core import TILEABLE_TYPE, OperatorType, TileableGraph
26
+ from ...core.operator.base import Operator
27
+ from ...serialization import PickleContainer
28
+ from ...utils import TypeDispatcher, no_default
29
+ from ..core import (
30
+ BUILTIN_ENGINE_SPE,
31
+ DAGCodeContext,
32
+ DAGCodeGenerator,
33
+ DagOperatorAdapter,
34
+ register_engine_codegen,
35
+ )
36
+ from .dataframe.udf import SpeUDF
37
+
38
+ _spe_op_adapter = TypeDispatcher()
39
+
40
+
41
+ def register_op_adapter(op_cls: Union[Type[OperatorType], List[Type[OperatorType]]]):
42
+ def wrapper(cls: Type[DagOperatorAdapter]):
43
+ op_classes = op_cls
44
+ if not isinstance(op_cls, list):
45
+ op_classes = [op_cls]
46
+ for op_class in op_classes:
47
+ _spe_op_adapter.register(op_class, cls)
48
+ return cls
49
+
50
+ return wrapper
51
+
52
+
53
+ class SPECodeContext(DAGCodeContext, ABC):
54
+ logger_var: str = "logger"
55
+
56
+ def __init__(self, session_id: str = None, subdag_id: str = None):
57
+ super().__init__(session_id, subdag_id)
58
+ self.imports = defaultdict(set)
59
+ self.import_froms = defaultdict(set)
60
+
61
+ def register_import(
62
+ self, mod: str, alias: Optional[str] = None, from_item: Optional[str] = None
63
+ ):
64
+ if from_item is None:
65
+ self.imports[mod].add(alias)
66
+ else:
67
+ self.import_froms[mod].add((from_item, alias))
68
+
69
+
70
+ class SPEOperatorAdapter(DagOperatorAdapter, ABC):
71
+ @abstractmethod
72
+ def generate_code(self, op: OperatorType, context: SPECodeContext) -> List[str]:
73
+ raise NotImplementedError
74
+
75
+ @classmethod
76
+ def generate_call_args_with_attributes(
77
+ cls,
78
+ op: OperatorType,
79
+ context: SPECodeContext,
80
+ *args: str,
81
+ skip_none: bool = False,
82
+ kw_keys: Optional[Iterable[str]] = None,
83
+ **kwargs: Optional[str],
84
+ ) -> str:
85
+ """
86
+ Generate the codes with simple copying attributes from the input operator to the
87
+ output Python call parameters. The absence of the attribute value will be
88
+ fallback to None.
89
+
90
+ Parameters
91
+ ----------
92
+ op: OperatorType
93
+ The operator instance.
94
+ context: SPECodeContext
95
+ The SPECodeContext instance.
96
+ *args:
97
+ The attribute names, which will remain the same name in the call parameter.
98
+ skip_none: bool
99
+ If True, will skip attributes with None value.
100
+ **kwargs:
101
+ The keyword arguments with string values, which will map to the name of the
102
+ call parameter.
103
+
104
+ e.g. old_name="new_name" will be __call__(new_name=getattr(obj, old_name))
105
+
106
+ Returns
107
+ -------
108
+ str :
109
+ The call args codes.
110
+ """
111
+ kw = {key: kwargs.get(key) for key in kw_keys or ()}
112
+ kw.update(kwargs)
113
+ str_args = [a for a in args if isinstance(a, str)]
114
+ all_args_dict = cls._collect_op_kwargs(
115
+ op, str_args + list(kw.keys()), skip_none=skip_none
116
+ )
117
+ args_list = [
118
+ all_args_dict.get(k) if isinstance(k, str) else op.inputs[k] for k in args
119
+ ]
120
+ kwargs_dict = {
121
+ kw.get(k) or k: all_args_dict.get(k)
122
+ for k in kw.keys()
123
+ if k in all_args_dict
124
+ }
125
+ return ", ".join(cls._translate_call_args(context, *args_list, **kwargs_dict))
126
+
127
+ @classmethod
128
+ def _collect_op_kwargs(
129
+ cls, op: OperatorType, args: List[str], skip_none: bool = True
130
+ ) -> Dict[str, str]:
131
+ kw = {}
132
+ for arg in args:
133
+ attr_val = getattr(op, arg, None)
134
+ if attr_val is not no_default and (not skip_none or attr_val is not None):
135
+ kw[arg] = attr_val
136
+ return kw
137
+
138
+ @classmethod
139
+ def _translate_call_args(
140
+ cls, context: SPECodeContext, *args, **kwargs
141
+ ) -> List[str]:
142
+ return [cls.translate_var(context, v) for v in args] + [
143
+ f"{k}={cls.translate_var(context, v)}" for k, v in kwargs.items()
144
+ ]
145
+
146
+ @classmethod
147
+ def translate_var(cls, context: SPECodeContext, val: Any) -> str:
148
+ """
149
+ Translate the val to a Python variable.
150
+
151
+ Parameters
152
+ ----------
153
+ context: SPECodeContext
154
+ The SPECodeContext instance.
155
+ val : Any
156
+ The value to be translated.
157
+
158
+ Returns
159
+ -------
160
+ str :
161
+ The var name which can be used in generated Python code.
162
+ """
163
+ if isinstance(val, TILEABLE_TYPE):
164
+ return context.get_input_tileable_variable(val)
165
+
166
+ if isinstance(val, (Callable, PickleContainer)) and not isinstance(
167
+ val, BaseOffset
168
+ ):
169
+ # TODO: handle used resources here
170
+ context.register_import("base64")
171
+ context.register_import("cloudpickle")
172
+ context.register_import("numpy", "np")
173
+ udf = SpeUDF(val)
174
+ context.register_udf(udf)
175
+ return udf.name
176
+
177
+ val_type = type(val)
178
+ if val_type is list:
179
+ vals_exp = [cls.translate_var(context, v) for v in val]
180
+ return f"[{', '.join(vals_exp)}]"
181
+
182
+ if val_type is tuple:
183
+ vals_exp = [cls.translate_var(context, v) for v in val]
184
+ if len(vals_exp) == 1:
185
+ # need a trailing space to make a tuple
186
+ vals_exp.append("")
187
+ return f"({', '.join(vals_exp).rstrip()})"
188
+
189
+ if val_type is slice:
190
+ return f"slice({val.start}, {val.stop}, {val.step})"
191
+
192
+ if val_type is ndarray:
193
+ return cls.translate_var(context, val.tolist())
194
+
195
+ if val_type is dict:
196
+ kvs = list()
197
+ for k, v in val.items():
198
+ kvs.append(
199
+ f"{cls.translate_var(context, k)}: {cls.translate_var(context, v)}"
200
+ )
201
+ return f"{{{', '.join(kvs)}}}"
202
+
203
+ if val_type is set:
204
+ keys = [cls.translate_var(context, k) for k in val]
205
+ return f"{{{', '.join(keys)}}}"
206
+
207
+ if val_type is pd.Timestamp:
208
+ context.register_import("pandas", "pd")
209
+ return f"pd.Timestamp({str(val)!r})"
210
+
211
+ if isinstance(val, npdtype):
212
+ context.register_import("numpy", "np")
213
+ return f"np.dtype({str(val)!r})"
214
+
215
+ return context.register_operator_constants(val)
216
+
217
+ def generate_pre_op_code(self, op: Operator, context: SPECodeContext) -> List[str]:
218
+ context.register_operator_constants(True, "running")
219
+ return ["if not running:\n raise RuntimeError('CANCELLED')"]
220
+
221
+ def gen_logging_code(
222
+ self, context: SPECodeContext, message: str, expressions: List[str]
223
+ ) -> str:
224
+ exp_str = ", ".join(expressions)
225
+ return f"{context.logger_var}.info('{message}', {exp_str})"
226
+
227
+ def gen_timecost_code(
228
+ self, context: SPECodeContext, phase_name: str, code_lines: List[str]
229
+ ) -> List[str]:
230
+ context.register_import("time")
231
+ codes = ["start_time = time.time()"]
232
+ codes.extend(code_lines)
233
+ codes.append(
234
+ f"{context.logger_var}.info('{phase_name} cost: %.2f s', time.time() - start_time)"
235
+ )
236
+ return codes
237
+
238
+
239
+ @register_engine_codegen
240
+ class SPECodeGenerator(DAGCodeGenerator):
241
+ _context: SPECodeContext
242
+
243
+ engine_type = BUILTIN_ENGINE_SPE
244
+ engine_priority = 0
245
+
246
+ def _init_context(self, session_id: str, subdag_id: str) -> SPECodeContext:
247
+ return SPECodeContext(session_id, subdag_id)
248
+
249
+ def _generate_delete_code(self, var_name: str) -> List[str]:
250
+ return [f"del {var_name}"]
251
+
252
+ @staticmethod
253
+ def _generate_import_code(module: str, alias: Optional[str] = None):
254
+ if alias is None:
255
+ return f"import {module}"
256
+ else:
257
+ return f"import {module} as {alias}"
258
+
259
+ @staticmethod
260
+ def _generate_import_from_code(
261
+ module: str, from_list: List[str], alias_list: Optional[List[str]] = None
262
+ ):
263
+ def build_from_str(from_str: str, alias: Optional[str]) -> str:
264
+ if alias is not None:
265
+ return f"{from_str} as {alias}"
266
+ else:
267
+ return from_str
268
+
269
+ alias_list = alias_list or itertools.repeat(None)
270
+ froms_str = ", ".join(
271
+ build_from_str(f, a) for f, a in zip(from_list, alias_list)
272
+ )
273
+ return f"from {module} import {froms_str}"
274
+
275
+ def get_op_adapter(self, op_type: Type[OperatorType]) -> Type[DagOperatorAdapter]:
276
+ return get_op_adapter(op_type)
277
+
278
+ def generate_code(self, dag: TileableGraph) -> List[str]:
279
+ from . import dataframe, tensor
280
+
281
+ del dataframe, tensor
282
+ main_codes = super().generate_code(dag)
283
+ import_codes = []
284
+ for mod, aliases in self._context.imports.items():
285
+ for alias in aliases:
286
+ import_codes.append(self._generate_import_code(mod, alias))
287
+ for mod, from_tuples in self._context.import_froms.items():
288
+ from_mods, aliases = [], []
289
+ for from_item, from_alias in from_tuples:
290
+ from_mods.append(from_item)
291
+ aliases.append(from_alias)
292
+ import_codes.append(
293
+ self._generate_import_from_code(mod, from_mods, aliases)
294
+ )
295
+ udf_codes = self._generate_udf_codes()
296
+ return import_codes + udf_codes + main_codes
297
+
298
+ def _generate_udf_codes(self) -> List[str]:
299
+ udf_codes = list()
300
+ for func in self.get_udfs():
301
+ udf_codes.extend(func.encoded_content)
302
+ udf_codes.append(f"{func.name} = pickled_data")
303
+ return udf_codes
304
+
305
+
306
+ def get_op_adapter(op_type: Type[OperatorType]) -> Type[SPEOperatorAdapter]:
307
+ return _spe_op_adapter.get_handler(op_type)
@@ -0,0 +1,37 @@
1
+ # Copyright 1999-2025 Alibaba Group Holding Ltd.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ from . import (
16
+ accessors,
17
+ arithmetic,
18
+ datasource,
19
+ datastore,
20
+ fetch,
21
+ groupby,
22
+ indexing,
23
+ merge,
24
+ misc,
25
+ missing,
26
+ reduction,
27
+ sort,
28
+ statistics,
29
+ tseries,
30
+ value_counts,
31
+ window,
32
+ )
33
+
34
+ try:
35
+ from . import _internal
36
+ except ImportError:
37
+ pass
@@ -0,0 +1,15 @@
1
+ # Copyright 1999-2025 Alibaba Group Holding Ltd.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ from . import base, dict_, list_