maxframe 1.3.0__cp38-cp38-win32.whl → 2.0.0b1__cp38-cp38-win32.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of maxframe might be problematic. Click here for more details.
- maxframe/_utils.cp38-win32.pyd +0 -0
- maxframe/_utils.pyi +21 -0
- maxframe/_utils.pyx +4 -3
- maxframe/codegen/__init__.py +27 -0
- maxframe/{codegen.py → codegen/core.py} +49 -43
- maxframe/codegen/spe/__init__.py +16 -0
- maxframe/codegen/spe/core.py +307 -0
- maxframe/codegen/spe/dataframe/__init__.py +37 -0
- maxframe/codegen/spe/dataframe/accessors/__init__.py +15 -0
- maxframe/codegen/spe/dataframe/accessors/base.py +53 -0
- maxframe/codegen/spe/dataframe/accessors/dict_.py +194 -0
- maxframe/codegen/spe/dataframe/accessors/list_.py +80 -0
- maxframe/codegen/spe/dataframe/arithmetic.py +84 -0
- maxframe/codegen/spe/dataframe/datasource.py +181 -0
- maxframe/codegen/spe/dataframe/datastore.py +204 -0
- maxframe/codegen/spe/dataframe/extensions.py +63 -0
- maxframe/codegen/spe/dataframe/fetch.py +26 -0
- maxframe/codegen/spe/dataframe/groupby.py +224 -0
- maxframe/codegen/spe/dataframe/indexing.py +238 -0
- maxframe/codegen/spe/dataframe/merge.py +73 -0
- maxframe/codegen/spe/dataframe/misc.py +286 -0
- maxframe/codegen/spe/dataframe/missing.py +64 -0
- maxframe/codegen/spe/dataframe/reduction.py +160 -0
- maxframe/codegen/spe/dataframe/sort.py +83 -0
- maxframe/codegen/spe/dataframe/statistics.py +46 -0
- maxframe/codegen/spe/dataframe/tests/__init__.py +13 -0
- maxframe/codegen/spe/dataframe/tests/accessors/__init__.py +13 -0
- maxframe/codegen/spe/dataframe/tests/accessors/test_base.py +33 -0
- maxframe/codegen/spe/dataframe/tests/accessors/test_dict.py +310 -0
- maxframe/codegen/spe/dataframe/tests/accessors/test_list.py +137 -0
- maxframe/codegen/spe/dataframe/tests/indexing/__init__.py +13 -0
- maxframe/codegen/spe/dataframe/tests/indexing/conftest.py +58 -0
- maxframe/codegen/spe/dataframe/tests/indexing/test_getitem.py +124 -0
- maxframe/codegen/spe/dataframe/tests/indexing/test_iloc.py +76 -0
- maxframe/codegen/spe/dataframe/tests/indexing/test_indexing.py +39 -0
- maxframe/codegen/spe/dataframe/tests/indexing/test_rename.py +51 -0
- maxframe/codegen/spe/dataframe/tests/indexing/test_reset_index.py +88 -0
- maxframe/codegen/spe/dataframe/tests/indexing/test_sample.py +45 -0
- maxframe/codegen/spe/dataframe/tests/indexing/test_set_axis.py +45 -0
- maxframe/codegen/spe/dataframe/tests/indexing/test_set_index.py +41 -0
- maxframe/codegen/spe/dataframe/tests/indexing/test_setitem.py +46 -0
- maxframe/codegen/spe/dataframe/tests/misc/__init__.py +13 -0
- maxframe/codegen/spe/dataframe/tests/misc/test_apply.py +133 -0
- maxframe/codegen/spe/dataframe/tests/misc/test_drop_duplicates.py +92 -0
- maxframe/codegen/spe/dataframe/tests/misc/test_misc.py +234 -0
- maxframe/codegen/spe/dataframe/tests/missing/__init__.py +13 -0
- maxframe/codegen/spe/dataframe/tests/missing/test_checkna.py +94 -0
- maxframe/codegen/spe/dataframe/tests/missing/test_dropna.py +50 -0
- maxframe/codegen/spe/dataframe/tests/missing/test_fillna.py +94 -0
- maxframe/codegen/spe/dataframe/tests/missing/test_replace.py +45 -0
- maxframe/codegen/spe/dataframe/tests/test_arithmetic.py +73 -0
- maxframe/codegen/spe/dataframe/tests/test_datasource.py +184 -0
- maxframe/codegen/spe/dataframe/tests/test_datastore.py +200 -0
- maxframe/codegen/spe/dataframe/tests/test_extensions.py +88 -0
- maxframe/codegen/spe/dataframe/tests/test_groupby.py +225 -0
- maxframe/codegen/spe/dataframe/tests/test_merge.py +400 -0
- maxframe/codegen/spe/dataframe/tests/test_reduction.py +104 -0
- maxframe/codegen/spe/dataframe/tests/test_sort.py +159 -0
- maxframe/codegen/spe/dataframe/tests/test_statistics.py +70 -0
- maxframe/codegen/spe/dataframe/tests/test_tseries.py +29 -0
- maxframe/codegen/spe/dataframe/tests/test_value_counts.py +60 -0
- maxframe/codegen/spe/dataframe/tests/test_window.py +69 -0
- maxframe/codegen/spe/dataframe/tseries.py +46 -0
- maxframe/codegen/spe/dataframe/udf.py +62 -0
- maxframe/codegen/spe/dataframe/value_counts.py +31 -0
- maxframe/codegen/spe/dataframe/window.py +65 -0
- maxframe/codegen/spe/learn/__init__.py +15 -0
- maxframe/codegen/spe/learn/contrib/__init__.py +15 -0
- maxframe/codegen/spe/learn/contrib/lightgbm.py +160 -0
- maxframe/codegen/spe/learn/contrib/models.py +41 -0
- maxframe/codegen/spe/learn/contrib/pytorch.py +49 -0
- maxframe/codegen/spe/learn/contrib/tests/__init__.py +13 -0
- maxframe/codegen/spe/learn/contrib/tests/test_lightgbm.py +123 -0
- maxframe/codegen/spe/learn/contrib/tests/test_models.py +41 -0
- maxframe/codegen/spe/learn/contrib/tests/test_pytorch.py +53 -0
- maxframe/codegen/spe/learn/contrib/tests/test_xgboost.py +98 -0
- maxframe/codegen/spe/learn/contrib/xgboost.py +152 -0
- maxframe/codegen/spe/learn/metrics/__init__.py +15 -0
- maxframe/codegen/spe/learn/metrics/_classification.py +120 -0
- maxframe/codegen/spe/learn/metrics/tests/__init__.py +13 -0
- maxframe/codegen/spe/learn/metrics/tests/test_classification.py +93 -0
- maxframe/codegen/spe/learn/model_selection/__init__.py +13 -0
- maxframe/codegen/spe/learn/model_selection/tests/__init__.py +13 -0
- maxframe/codegen/spe/learn/model_selection/tests/test_split.py +41 -0
- maxframe/codegen/spe/learn/preprocessing/__init__.py +15 -0
- maxframe/codegen/spe/learn/preprocessing/_data.py +37 -0
- maxframe/codegen/spe/learn/preprocessing/_label.py +47 -0
- maxframe/codegen/spe/learn/preprocessing/tests/__init__.py +13 -0
- maxframe/codegen/spe/learn/preprocessing/tests/test_data.py +31 -0
- maxframe/codegen/spe/learn/preprocessing/tests/test_label.py +43 -0
- maxframe/codegen/spe/learn/utils/__init__.py +15 -0
- maxframe/codegen/spe/learn/utils/checks.py +55 -0
- maxframe/codegen/spe/learn/utils/multiclass.py +60 -0
- maxframe/codegen/spe/learn/utils/shuffle.py +85 -0
- maxframe/codegen/spe/learn/utils/sparsefuncs.py +35 -0
- maxframe/codegen/spe/learn/utils/tests/__init__.py +13 -0
- maxframe/codegen/spe/learn/utils/tests/test_checks.py +48 -0
- maxframe/codegen/spe/learn/utils/tests/test_multiclass.py +52 -0
- maxframe/codegen/spe/learn/utils/tests/test_shuffle.py +50 -0
- maxframe/codegen/spe/learn/utils/tests/test_sparsefuncs.py +34 -0
- maxframe/codegen/spe/learn/utils/tests/test_validation.py +44 -0
- maxframe/codegen/spe/learn/utils/validation.py +35 -0
- maxframe/codegen/spe/objects.py +26 -0
- maxframe/codegen/spe/remote.py +29 -0
- maxframe/codegen/spe/tensor/__init__.py +28 -0
- maxframe/codegen/spe/tensor/arithmetic.py +95 -0
- maxframe/codegen/spe/tensor/core.py +41 -0
- maxframe/codegen/spe/tensor/datasource.py +165 -0
- maxframe/codegen/spe/tensor/extensions.py +35 -0
- maxframe/codegen/spe/tensor/fetch.py +26 -0
- maxframe/codegen/spe/tensor/indexing.py +63 -0
- maxframe/codegen/spe/tensor/linalg.py +63 -0
- maxframe/codegen/spe/tensor/merge.py +31 -0
- maxframe/codegen/spe/tensor/misc.py +121 -0
- maxframe/codegen/spe/tensor/random.py +29 -0
- maxframe/codegen/spe/tensor/reduction.py +39 -0
- maxframe/codegen/spe/tensor/reshape.py +26 -0
- maxframe/codegen/spe/tensor/sort.py +42 -0
- maxframe/codegen/spe/tensor/special.py +35 -0
- maxframe/codegen/spe/tensor/statistics.py +24 -0
- maxframe/codegen/spe/tensor/tests/__init__.py +13 -0
- maxframe/codegen/spe/tensor/tests/test_arithmetic.py +103 -0
- maxframe/codegen/spe/tensor/tests/test_datasource.py +99 -0
- maxframe/codegen/spe/tensor/tests/test_extensions.py +37 -0
- maxframe/codegen/spe/tensor/tests/test_indexing.py +44 -0
- maxframe/codegen/spe/tensor/tests/test_linalg.py +38 -0
- maxframe/codegen/spe/tensor/tests/test_merge.py +28 -0
- maxframe/codegen/spe/tensor/tests/test_misc.py +94 -0
- maxframe/codegen/spe/tensor/tests/test_random.py +55 -0
- maxframe/codegen/spe/tensor/tests/test_reduction.py +65 -0
- maxframe/codegen/spe/tensor/tests/test_reshape.py +39 -0
- maxframe/codegen/spe/tensor/tests/test_sort.py +49 -0
- maxframe/codegen/spe/tensor/tests/test_special.py +28 -0
- maxframe/codegen/spe/tensor/tests/test_statistics.py +29 -0
- maxframe/codegen/spe/tests/__init__.py +13 -0
- maxframe/codegen/spe/tests/test_remote.py +29 -0
- maxframe/codegen/spe/tests/test_spe_codegen.py +141 -0
- maxframe/codegen/spe/utils.py +54 -0
- maxframe/codegen/tests/__init__.py +13 -0
- maxframe/{tests → codegen/tests}/test_codegen.py +3 -5
- maxframe/config/__init__.py +1 -1
- maxframe/config/config.py +50 -23
- maxframe/config/tests/test_config.py +4 -12
- maxframe/config/validators.py +5 -0
- maxframe/conftest.py +38 -10
- maxframe/core/__init__.py +1 -0
- maxframe/core/context.py +110 -0
- maxframe/core/entity/__init__.py +1 -0
- maxframe/core/entity/core.py +0 -7
- maxframe/core/entity/objects.py +19 -5
- maxframe/core/entity/output_types.py +11 -0
- maxframe/core/entity/tests/test_objects.py +11 -12
- maxframe/core/entity/tileables.py +3 -1
- maxframe/core/entity/utils.py +15 -0
- maxframe/core/graph/__init__.py +6 -1
- maxframe/core/graph/builder/base.py +5 -1
- maxframe/core/graph/core.cp38-win32.pyd +0 -0
- maxframe/core/graph/core.pyx +17 -6
- maxframe/core/graph/entity.py +18 -6
- maxframe/core/operator/__init__.py +8 -3
- maxframe/core/operator/base.py +35 -12
- maxframe/core/operator/core.py +37 -14
- maxframe/core/operator/fetch.py +5 -18
- maxframe/core/operator/objects.py +0 -20
- maxframe/core/operator/shuffle.py +6 -72
- maxframe/dataframe/__init__.py +1 -0
- maxframe/dataframe/accessors/datetime_/core.py +7 -4
- maxframe/dataframe/accessors/string_/core.py +9 -6
- maxframe/dataframe/arithmetic/core.py +31 -20
- maxframe/dataframe/arithmetic/tests/test_arithmetic.py +6 -0
- maxframe/dataframe/core.py +98 -91
- maxframe/dataframe/datasource/core.py +8 -1
- maxframe/dataframe/datasource/date_range.py +8 -0
- maxframe/dataframe/datasource/from_index.py +9 -5
- maxframe/dataframe/datasource/from_records.py +9 -2
- maxframe/dataframe/datasource/from_tensor.py +32 -21
- maxframe/dataframe/datasource/read_csv.py +8 -2
- maxframe/dataframe/datasource/read_odps_query.py +33 -3
- maxframe/dataframe/datasource/read_odps_table.py +20 -5
- maxframe/dataframe/datasource/read_parquet.py +8 -3
- maxframe/dataframe/datasource/tests/test_datasource.py +33 -0
- maxframe/dataframe/datastore/tests/test_to_odps.py +52 -1
- maxframe/dataframe/datastore/to_csv.py +7 -3
- maxframe/dataframe/datastore/to_odps.py +42 -6
- maxframe/dataframe/extensions/__init__.py +6 -1
- maxframe/dataframe/extensions/apply_chunk.py +96 -136
- maxframe/dataframe/extensions/flatjson.py +3 -2
- maxframe/dataframe/extensions/flatmap.py +15 -7
- maxframe/dataframe/fetch/core.py +12 -1
- maxframe/dataframe/groupby/__init__.py +7 -0
- maxframe/dataframe/groupby/aggregation.py +62 -9
- maxframe/dataframe/groupby/apply.py +50 -74
- maxframe/dataframe/groupby/apply_chunk.py +393 -0
- maxframe/dataframe/groupby/core.py +80 -17
- maxframe/dataframe/groupby/extensions.py +26 -0
- maxframe/dataframe/groupby/fill.py +9 -4
- maxframe/dataframe/groupby/sample.py +7 -7
- maxframe/dataframe/groupby/tests/test_groupby.py +3 -3
- maxframe/dataframe/groupby/transform.py +57 -54
- maxframe/dataframe/indexing/align.py +7 -6
- maxframe/dataframe/indexing/getitem.py +9 -8
- maxframe/dataframe/indexing/iloc.py +28 -23
- maxframe/dataframe/indexing/insert.py +7 -3
- maxframe/dataframe/indexing/loc.py +9 -8
- maxframe/dataframe/indexing/reindex.py +36 -30
- maxframe/dataframe/indexing/rename_axis.py +18 -10
- maxframe/dataframe/indexing/reset_index.py +0 -2
- maxframe/dataframe/indexing/sample.py +13 -9
- maxframe/dataframe/indexing/set_axis.py +9 -6
- maxframe/dataframe/indexing/setitem.py +8 -5
- maxframe/dataframe/indexing/where.py +12 -9
- maxframe/dataframe/merge/__init__.py +0 -1
- maxframe/dataframe/merge/concat.py +10 -31
- maxframe/dataframe/merge/merge.py +2 -24
- maxframe/dataframe/misc/__init__.py +6 -0
- maxframe/dataframe/misc/_duplicate.py +7 -3
- maxframe/dataframe/misc/apply.py +106 -139
- maxframe/dataframe/misc/astype.py +3 -2
- maxframe/dataframe/misc/case_when.py +11 -7
- maxframe/dataframe/misc/cut.py +11 -10
- maxframe/dataframe/misc/describe.py +7 -3
- maxframe/dataframe/misc/drop.py +13 -11
- maxframe/dataframe/misc/eval.py +0 -2
- maxframe/dataframe/misc/get_dummies.py +78 -49
- maxframe/dataframe/misc/isin.py +13 -10
- maxframe/dataframe/misc/map.py +21 -6
- maxframe/dataframe/misc/melt.py +8 -1
- maxframe/dataframe/misc/pivot.py +232 -0
- maxframe/dataframe/misc/pivot_table.py +52 -40
- maxframe/dataframe/misc/rechunk.py +59 -0
- maxframe/dataframe/misc/shift.py +7 -4
- maxframe/dataframe/misc/stack.py +5 -3
- maxframe/dataframe/misc/tests/test_misc.py +167 -1
- maxframe/dataframe/misc/transform.py +63 -65
- maxframe/dataframe/misc/value_counts.py +7 -4
- maxframe/dataframe/missing/dropna.py +16 -7
- maxframe/dataframe/missing/fillna.py +18 -10
- maxframe/dataframe/missing/replace.py +10 -6
- maxframe/dataframe/missing/tests/test_missing.py +2 -2
- maxframe/dataframe/operators.py +1 -27
- maxframe/dataframe/reduction/aggregation.py +128 -3
- maxframe/dataframe/reduction/core.py +20 -6
- maxframe/dataframe/reduction/median.py +1 -1
- maxframe/dataframe/reduction/tests/test_reduction.py +33 -0
- maxframe/dataframe/reduction/unique.py +53 -7
- maxframe/dataframe/statistics/corr.py +9 -6
- maxframe/dataframe/statistics/quantile.py +9 -6
- maxframe/dataframe/tseries/to_datetime.py +6 -4
- maxframe/dataframe/utils.py +219 -31
- maxframe/dataframe/window/rolling.py +7 -4
- maxframe/env.py +1 -0
- maxframe/errors.py +9 -0
- maxframe/extension.py +13 -2
- maxframe/io/objects/core.py +67 -51
- maxframe/io/objects/tensor.py +73 -17
- maxframe/io/objects/tests/test_object_io.py +8 -55
- maxframe/io/odpsio/arrow.py +15 -2
- maxframe/io/odpsio/schema.py +43 -13
- maxframe/io/odpsio/tableio.py +63 -11
- maxframe/io/odpsio/tests/test_arrow.py +1 -2
- maxframe/io/odpsio/tests/test_schema.py +114 -1
- maxframe/io/odpsio/tests/test_tableio.py +42 -0
- maxframe/io/odpsio/tests/test_volumeio.py +22 -48
- maxframe/learn/__init__.py +2 -2
- maxframe/learn/contrib/__init__.py +2 -2
- maxframe/learn/contrib/graph/connected_components.py +2 -1
- maxframe/learn/contrib/lightgbm/__init__.py +33 -0
- maxframe/learn/contrib/lightgbm/_predict.py +138 -0
- maxframe/learn/contrib/lightgbm/_train.py +163 -0
- maxframe/learn/contrib/lightgbm/callback.py +114 -0
- maxframe/learn/contrib/lightgbm/classifier.py +199 -0
- maxframe/learn/contrib/lightgbm/core.py +372 -0
- maxframe/learn/contrib/lightgbm/dataset.py +153 -0
- maxframe/learn/contrib/lightgbm/regressor.py +29 -0
- maxframe/learn/contrib/lightgbm/tests/__init__.py +13 -0
- maxframe/learn/contrib/lightgbm/tests/test_callback.py +58 -0
- maxframe/learn/contrib/llm/models/dashscope.py +34 -0
- maxframe/learn/contrib/llm/models/managed.py +15 -0
- maxframe/learn/contrib/llm/multi_modal.py +92 -0
- maxframe/learn/contrib/llm/text.py +21 -5
- maxframe/learn/contrib/models.py +38 -9
- maxframe/learn/contrib/utils.py +55 -0
- maxframe/learn/contrib/xgboost/callback.py +86 -0
- maxframe/learn/contrib/xgboost/classifier.py +26 -30
- maxframe/learn/contrib/xgboost/core.py +53 -42
- maxframe/learn/contrib/xgboost/dmatrix.py +19 -12
- maxframe/learn/contrib/xgboost/predict.py +13 -8
- maxframe/learn/contrib/xgboost/regressor.py +28 -27
- maxframe/learn/contrib/xgboost/tests/test_callback.py +41 -0
- maxframe/learn/contrib/xgboost/train.py +59 -16
- maxframe/learn/core.py +252 -0
- maxframe/learn/datasets/__init__.py +20 -0
- maxframe/learn/datasets/samples_generator.py +628 -0
- maxframe/learn/linear_model/__init__.py +15 -0
- maxframe/learn/linear_model/_base.py +163 -0
- maxframe/learn/linear_model/_lin_reg.py +175 -0
- maxframe/learn/metrics/__init__.py +25 -0
- maxframe/learn/metrics/_check_targets.py +95 -0
- maxframe/learn/metrics/_classification.py +1121 -0
- maxframe/learn/metrics/_regression.py +256 -0
- maxframe/learn/model_selection/__init__.py +15 -0
- maxframe/learn/model_selection/_split.py +451 -0
- maxframe/learn/model_selection/tests/__init__.py +13 -0
- maxframe/learn/model_selection/tests/test_split.py +156 -0
- maxframe/learn/preprocessing/__init__.py +16 -0
- maxframe/learn/preprocessing/_data/__init__.py +17 -0
- maxframe/learn/preprocessing/_data/min_max_scaler.py +390 -0
- maxframe/learn/preprocessing/_data/normalize.py +127 -0
- maxframe/learn/preprocessing/_data/standard_scaler.py +503 -0
- maxframe/learn/preprocessing/_data/utils.py +79 -0
- maxframe/learn/preprocessing/_label/__init__.py +16 -0
- maxframe/learn/preprocessing/_label/_label_binarizer.py +599 -0
- maxframe/learn/preprocessing/_label/_label_encoder.py +174 -0
- maxframe/learn/utils/__init__.py +4 -0
- maxframe/learn/utils/_encode.py +314 -0
- maxframe/learn/utils/checks.py +161 -0
- maxframe/learn/utils/core.py +33 -0
- maxframe/learn/utils/extmath.py +176 -0
- maxframe/learn/utils/multiclass.py +292 -0
- maxframe/learn/utils/shuffle.py +114 -0
- maxframe/learn/utils/sparsefuncs.py +87 -0
- maxframe/learn/utils/validation.py +775 -0
- maxframe/lib/__init__.py +0 -2
- maxframe/lib/compat.py +145 -0
- maxframe/lib/filesystem/_oss_lib/glob.py +1 -1
- maxframe/lib/mmh3.cp38-win32.pyd +0 -0
- maxframe/lib/sparse/__init__.py +10 -15
- maxframe/lib/sparse/array.py +45 -33
- maxframe/lib/sparse/core.py +0 -2
- maxframe/lib/sparse/linalg.py +31 -0
- maxframe/lib/sparse/matrix.py +5 -2
- maxframe/lib/sparse/tests/__init__.py +0 -2
- maxframe/lib/sparse/tests/test_sparse.py +53 -53
- maxframe/lib/sparse/vector.py +0 -2
- maxframe/mixin.py +59 -2
- maxframe/opcodes.py +13 -5
- maxframe/protocol.py +67 -14
- maxframe/remote/core.py +16 -14
- maxframe/remote/run_script.py +6 -3
- maxframe/serialization/__init__.py +2 -0
- maxframe/serialization/core.cp38-win32.pyd +0 -0
- maxframe/serialization/core.pxd +3 -0
- maxframe/serialization/core.pyi +3 -1
- maxframe/serialization/core.pyx +82 -4
- maxframe/serialization/pandas.py +5 -1
- maxframe/serialization/serializables/core.py +6 -5
- maxframe/serialization/serializables/field.py +2 -2
- maxframe/serialization/serializables/tests/test_field_type.py +3 -5
- maxframe/serialization/tests/test_serial.py +27 -0
- maxframe/session.py +4 -71
- maxframe/sperunner.py +165 -0
- maxframe/tensor/__init__.py +35 -2
- maxframe/tensor/arithmetic/__init__.py +2 -4
- maxframe/tensor/arithmetic/abs.py +0 -2
- maxframe/tensor/arithmetic/absolute.py +0 -2
- maxframe/tensor/arithmetic/add.py +34 -4
- maxframe/tensor/arithmetic/angle.py +0 -2
- maxframe/tensor/arithmetic/arccos.py +1 -4
- maxframe/tensor/arithmetic/arccosh.py +1 -3
- maxframe/tensor/arithmetic/arcsin.py +0 -2
- maxframe/tensor/arithmetic/arcsinh.py +0 -2
- maxframe/tensor/arithmetic/arctan.py +0 -2
- maxframe/tensor/arithmetic/arctan2.py +0 -2
- maxframe/tensor/arithmetic/arctanh.py +0 -2
- maxframe/tensor/arithmetic/around.py +0 -2
- maxframe/tensor/arithmetic/bitand.py +0 -2
- maxframe/tensor/arithmetic/bitor.py +1 -3
- maxframe/tensor/arithmetic/bitxor.py +1 -3
- maxframe/tensor/arithmetic/cbrt.py +0 -2
- maxframe/tensor/arithmetic/ceil.py +0 -2
- maxframe/tensor/arithmetic/clip.py +13 -13
- maxframe/tensor/arithmetic/conj.py +0 -2
- maxframe/tensor/arithmetic/copysign.py +0 -2
- maxframe/tensor/arithmetic/core.py +47 -39
- maxframe/tensor/arithmetic/cos.py +1 -3
- maxframe/tensor/arithmetic/cosh.py +0 -2
- maxframe/tensor/arithmetic/deg2rad.py +0 -2
- maxframe/tensor/arithmetic/degrees.py +0 -2
- maxframe/tensor/arithmetic/divide.py +0 -2
- maxframe/tensor/arithmetic/equal.py +0 -2
- maxframe/tensor/arithmetic/exp.py +1 -3
- maxframe/tensor/arithmetic/exp2.py +0 -2
- maxframe/tensor/arithmetic/expm1.py +0 -2
- maxframe/tensor/arithmetic/fabs.py +0 -2
- maxframe/tensor/arithmetic/fix.py +0 -2
- maxframe/tensor/arithmetic/float_power.py +0 -2
- maxframe/tensor/arithmetic/floor.py +0 -2
- maxframe/tensor/arithmetic/floordiv.py +0 -2
- maxframe/tensor/arithmetic/fmax.py +0 -2
- maxframe/tensor/arithmetic/fmin.py +0 -2
- maxframe/tensor/arithmetic/fmod.py +0 -2
- maxframe/tensor/arithmetic/frexp.py +6 -2
- maxframe/tensor/arithmetic/greater.py +0 -2
- maxframe/tensor/arithmetic/greater_equal.py +0 -2
- maxframe/tensor/arithmetic/hypot.py +0 -2
- maxframe/tensor/arithmetic/i0.py +1 -3
- maxframe/tensor/arithmetic/imag.py +0 -2
- maxframe/tensor/arithmetic/invert.py +1 -3
- maxframe/tensor/arithmetic/isclose.py +0 -2
- maxframe/tensor/arithmetic/iscomplex.py +0 -2
- maxframe/tensor/arithmetic/isfinite.py +1 -3
- maxframe/tensor/arithmetic/isinf.py +0 -2
- maxframe/tensor/arithmetic/isnan.py +0 -2
- maxframe/tensor/arithmetic/isreal.py +0 -2
- maxframe/tensor/arithmetic/ldexp.py +0 -2
- maxframe/tensor/arithmetic/less.py +0 -2
- maxframe/tensor/arithmetic/less_equal.py +0 -2
- maxframe/tensor/arithmetic/log.py +1 -3
- maxframe/tensor/arithmetic/log10.py +1 -3
- maxframe/tensor/arithmetic/log1p.py +1 -3
- maxframe/tensor/arithmetic/log2.py +1 -3
- maxframe/tensor/arithmetic/logaddexp.py +0 -2
- maxframe/tensor/arithmetic/logaddexp2.py +0 -2
- maxframe/tensor/arithmetic/logical_and.py +0 -2
- maxframe/tensor/arithmetic/logical_not.py +1 -3
- maxframe/tensor/arithmetic/logical_or.py +0 -2
- maxframe/tensor/arithmetic/logical_xor.py +0 -2
- maxframe/tensor/arithmetic/lshift.py +0 -2
- maxframe/tensor/arithmetic/maximum.py +0 -2
- maxframe/tensor/arithmetic/minimum.py +0 -2
- maxframe/tensor/arithmetic/mod.py +0 -2
- maxframe/tensor/arithmetic/modf.py +6 -2
- maxframe/tensor/arithmetic/multiply.py +37 -4
- maxframe/tensor/arithmetic/nan_to_num.py +0 -2
- maxframe/tensor/arithmetic/negative.py +0 -2
- maxframe/tensor/arithmetic/nextafter.py +0 -2
- maxframe/tensor/arithmetic/not_equal.py +0 -2
- maxframe/tensor/arithmetic/positive.py +0 -2
- maxframe/tensor/arithmetic/power.py +0 -2
- maxframe/tensor/arithmetic/rad2deg.py +0 -2
- maxframe/tensor/arithmetic/radians.py +0 -2
- maxframe/tensor/arithmetic/real.py +0 -2
- maxframe/tensor/arithmetic/reciprocal.py +5 -3
- maxframe/tensor/arithmetic/rint.py +1 -3
- maxframe/tensor/arithmetic/rshift.py +0 -2
- maxframe/tensor/arithmetic/setimag.py +0 -2
- maxframe/tensor/arithmetic/setreal.py +0 -2
- maxframe/tensor/arithmetic/sign.py +0 -2
- maxframe/tensor/arithmetic/signbit.py +0 -2
- maxframe/tensor/arithmetic/sin.py +0 -2
- maxframe/tensor/arithmetic/sinc.py +1 -3
- maxframe/tensor/arithmetic/sinh.py +0 -2
- maxframe/tensor/arithmetic/spacing.py +0 -2
- maxframe/tensor/arithmetic/sqrt.py +0 -2
- maxframe/tensor/arithmetic/square.py +0 -2
- maxframe/tensor/arithmetic/subtract.py +4 -2
- maxframe/tensor/arithmetic/tan.py +0 -2
- maxframe/tensor/arithmetic/tanh.py +0 -2
- maxframe/tensor/arithmetic/tests/__init__.py +0 -2
- maxframe/tensor/arithmetic/tests/test_arithmetic.py +43 -9
- maxframe/tensor/arithmetic/truediv.py +0 -2
- maxframe/tensor/arithmetic/trunc.py +0 -2
- maxframe/tensor/arithmetic/utils.py +32 -6
- maxframe/tensor/array_utils.py +3 -25
- maxframe/tensor/core.py +6 -6
- maxframe/tensor/datasource/__init__.py +10 -2
- maxframe/tensor/datasource/arange.py +0 -2
- maxframe/tensor/datasource/array.py +3 -22
- maxframe/tensor/datasource/core.py +15 -10
- maxframe/tensor/datasource/diag.py +140 -0
- maxframe/tensor/datasource/diagflat.py +69 -0
- maxframe/tensor/datasource/empty.py +0 -2
- maxframe/tensor/datasource/eye.py +95 -0
- maxframe/tensor/datasource/from_dataframe.py +0 -2
- maxframe/tensor/datasource/from_dense.py +0 -17
- maxframe/tensor/datasource/from_sparse.py +0 -2
- maxframe/tensor/datasource/full.py +0 -2
- maxframe/tensor/datasource/identity.py +54 -0
- maxframe/tensor/datasource/indices.py +115 -0
- maxframe/tensor/datasource/linspace.py +140 -0
- maxframe/tensor/datasource/meshgrid.py +135 -0
- maxframe/tensor/datasource/ones.py +8 -3
- maxframe/tensor/datasource/tests/test_datasource.py +32 -1
- maxframe/tensor/datasource/tri_array.py +107 -0
- maxframe/tensor/datasource/zeros.py +7 -3
- maxframe/tensor/extensions/__init__.py +31 -0
- maxframe/tensor/extensions/accessor.py +25 -0
- maxframe/tensor/extensions/apply_chunk.py +137 -0
- maxframe/tensor/indexing/__init__.py +1 -1
- maxframe/tensor/indexing/choose.py +8 -6
- maxframe/tensor/indexing/compress.py +0 -2
- maxframe/tensor/indexing/extract.py +0 -2
- maxframe/tensor/indexing/fill_diagonal.py +9 -6
- maxframe/tensor/indexing/flatnonzero.py +1 -3
- maxframe/tensor/indexing/getitem.py +10 -43
- maxframe/tensor/indexing/nonzero.py +2 -4
- maxframe/tensor/indexing/setitem.py +19 -9
- maxframe/tensor/indexing/slice.py +6 -3
- maxframe/tensor/indexing/take.py +0 -2
- maxframe/tensor/indexing/tests/__init__.py +0 -2
- maxframe/tensor/indexing/tests/test_indexing.py +0 -2
- maxframe/tensor/indexing/unravel_index.py +6 -6
- maxframe/tensor/lib/__init__.py +16 -0
- maxframe/tensor/lib/index_tricks.py +404 -0
- maxframe/tensor/linalg/__init__.py +36 -0
- maxframe/tensor/linalg/dot.py +145 -0
- maxframe/tensor/linalg/inner.py +36 -0
- maxframe/tensor/linalg/inv.py +83 -0
- maxframe/tensor/linalg/lu.py +115 -0
- maxframe/tensor/linalg/matmul.py +225 -0
- maxframe/tensor/linalg/qr.py +124 -0
- maxframe/tensor/linalg/solve_triangular.py +103 -0
- maxframe/tensor/linalg/svd.py +167 -0
- maxframe/tensor/linalg/tensordot.py +213 -0
- maxframe/tensor/linalg/vdot.py +73 -0
- maxframe/tensor/merge/__init__.py +4 -0
- maxframe/tensor/merge/append.py +74 -0
- maxframe/tensor/merge/column_stack.py +63 -0
- maxframe/tensor/merge/concatenate.py +3 -2
- maxframe/tensor/merge/dstack.py +71 -0
- maxframe/tensor/merge/hstack.py +70 -0
- maxframe/tensor/merge/stack.py +0 -2
- maxframe/tensor/merge/tests/test_merge.py +0 -2
- maxframe/tensor/misc/__init__.py +18 -5
- maxframe/tensor/misc/astype.py +10 -8
- maxframe/tensor/misc/broadcast_to.py +1 -1
- maxframe/tensor/misc/copy.py +64 -0
- maxframe/tensor/misc/diff.py +115 -0
- maxframe/tensor/misc/flatten.py +63 -0
- maxframe/tensor/misc/in1d.py +94 -0
- maxframe/tensor/misc/isin.py +130 -0
- maxframe/tensor/misc/ndim.py +53 -0
- maxframe/tensor/misc/ravel.py +0 -2
- maxframe/tensor/misc/repeat.py +129 -0
- maxframe/tensor/misc/searchsorted.py +147 -0
- maxframe/tensor/misc/setdiff1d.py +58 -0
- maxframe/tensor/misc/squeeze.py +117 -0
- maxframe/tensor/misc/swapaxes.py +113 -0
- maxframe/tensor/misc/tests/test_misc.py +0 -2
- maxframe/tensor/misc/transpose.py +8 -4
- maxframe/tensor/misc/trapezoid.py +123 -0
- maxframe/tensor/misc/unique.py +0 -1
- maxframe/tensor/misc/where.py +10 -8
- maxframe/tensor/operators.py +0 -34
- maxframe/tensor/random/__init__.py +3 -5
- maxframe/tensor/random/binomial.py +0 -2
- maxframe/tensor/random/bytes.py +0 -2
- maxframe/tensor/random/chisquare.py +0 -2
- maxframe/tensor/random/choice.py +9 -8
- maxframe/tensor/random/core.py +20 -5
- maxframe/tensor/random/dirichlet.py +0 -2
- maxframe/tensor/random/exponential.py +0 -2
- maxframe/tensor/random/f.py +2 -4
- maxframe/tensor/random/gamma.py +0 -2
- maxframe/tensor/random/geometric.py +0 -2
- maxframe/tensor/random/gumbel.py +0 -2
- maxframe/tensor/random/hypergeometric.py +0 -2
- maxframe/tensor/random/laplace.py +2 -4
- maxframe/tensor/random/logistic.py +0 -2
- maxframe/tensor/random/lognormal.py +0 -2
- maxframe/tensor/random/logseries.py +0 -2
- maxframe/tensor/random/multinomial.py +0 -2
- maxframe/tensor/random/multivariate_normal.py +0 -2
- maxframe/tensor/random/negative_binomial.py +0 -2
- maxframe/tensor/random/noncentral_chisquare.py +0 -2
- maxframe/tensor/random/noncentral_f.py +1 -3
- maxframe/tensor/random/normal.py +0 -2
- maxframe/tensor/random/pareto.py +0 -2
- maxframe/tensor/random/permutation.py +6 -3
- maxframe/tensor/random/poisson.py +0 -2
- maxframe/tensor/random/power.py +0 -2
- maxframe/tensor/random/rand.py +0 -2
- maxframe/tensor/random/randint.py +0 -2
- maxframe/tensor/random/randn.py +0 -2
- maxframe/tensor/random/random_integers.py +0 -2
- maxframe/tensor/random/random_sample.py +0 -2
- maxframe/tensor/random/rayleigh.py +0 -2
- maxframe/tensor/random/standard_cauchy.py +0 -2
- maxframe/tensor/random/standard_exponential.py +0 -2
- maxframe/tensor/random/standard_gamma.py +0 -2
- maxframe/tensor/random/standard_normal.py +0 -2
- maxframe/tensor/random/standard_t.py +0 -2
- maxframe/tensor/random/tests/__init__.py +0 -2
- maxframe/tensor/random/tests/test_random.py +0 -2
- maxframe/tensor/random/triangular.py +0 -2
- maxframe/tensor/random/uniform.py +0 -2
- maxframe/tensor/random/vonmises.py +0 -2
- maxframe/tensor/random/wald.py +0 -2
- maxframe/tensor/random/weibull.py +0 -2
- maxframe/tensor/random/zipf.py +0 -2
- maxframe/tensor/reduction/__init__.py +0 -2
- maxframe/tensor/reduction/all.py +0 -2
- maxframe/tensor/reduction/allclose.py +0 -2
- maxframe/tensor/reduction/any.py +0 -2
- maxframe/tensor/reduction/argmax.py +1 -3
- maxframe/tensor/reduction/argmin.py +1 -3
- maxframe/tensor/reduction/array_equal.py +0 -2
- maxframe/tensor/reduction/core.py +0 -2
- maxframe/tensor/reduction/count_nonzero.py +0 -2
- maxframe/tensor/reduction/cumprod.py +0 -2
- maxframe/tensor/reduction/cumsum.py +0 -2
- maxframe/tensor/reduction/max.py +0 -2
- maxframe/tensor/reduction/mean.py +0 -2
- maxframe/tensor/reduction/min.py +0 -2
- maxframe/tensor/reduction/nanargmax.py +0 -2
- maxframe/tensor/reduction/nanargmin.py +0 -2
- maxframe/tensor/reduction/nancumprod.py +0 -2
- maxframe/tensor/reduction/nancumsum.py +0 -2
- maxframe/tensor/reduction/nanmax.py +0 -2
- maxframe/tensor/reduction/nanmean.py +0 -2
- maxframe/tensor/reduction/nanmin.py +0 -2
- maxframe/tensor/reduction/nanprod.py +0 -2
- maxframe/tensor/reduction/nanstd.py +0 -2
- maxframe/tensor/reduction/nansum.py +0 -2
- maxframe/tensor/reduction/nanvar.py +0 -2
- maxframe/tensor/reduction/prod.py +0 -2
- maxframe/tensor/reduction/std.py +0 -2
- maxframe/tensor/reduction/sum.py +0 -2
- maxframe/tensor/reduction/tests/test_reduction.py +1 -4
- maxframe/tensor/reduction/var.py +0 -2
- maxframe/tensor/reshape/__init__.py +0 -2
- maxframe/tensor/reshape/reshape.py +6 -5
- maxframe/tensor/reshape/tests/__init__.py +0 -2
- maxframe/tensor/reshape/tests/test_reshape.py +0 -2
- maxframe/tensor/sort/__init__.py +16 -0
- maxframe/tensor/sort/argsort.py +150 -0
- maxframe/tensor/sort/sort.py +295 -0
- maxframe/tensor/special/__init__.py +37 -0
- maxframe/tensor/special/core.py +38 -0
- maxframe/tensor/special/misc.py +142 -0
- maxframe/tensor/special/statistical.py +56 -0
- maxframe/tensor/statistics/__init__.py +5 -0
- maxframe/tensor/statistics/average.py +143 -0
- maxframe/tensor/statistics/bincount.py +133 -0
- maxframe/tensor/statistics/quantile.py +10 -8
- maxframe/tensor/ufunc/__init__.py +0 -2
- maxframe/tensor/ufunc/ufunc.py +0 -2
- maxframe/tensor/utils.py +21 -3
- maxframe/tests/test_protocol.py +3 -3
- maxframe/tests/test_utils.py +210 -1
- maxframe/tests/utils.py +67 -1
- maxframe/udf.py +76 -6
- maxframe/utils.py +418 -17
- {maxframe-1.3.0.dist-info → maxframe-2.0.0b1.dist-info}/METADATA +5 -1
- maxframe-2.0.0b1.dist-info/RECORD +939 -0
- maxframe_client/clients/framedriver.py +19 -3
- maxframe_client/fetcher.py +113 -6
- maxframe_client/session/odps.py +173 -38
- maxframe_client/session/task.py +3 -1
- maxframe_client/tests/test_session.py +41 -5
- maxframe-1.3.0.dist-info/RECORD +0 -705
- {maxframe-1.3.0.dist-info → maxframe-2.0.0b1.dist-info}/WHEEL +0 -0
- {maxframe-1.3.0.dist-info → maxframe-2.0.0b1.dist-info}/top_level.txt +0 -0
maxframe/_utils.cp38-win32.pyd
CHANGED
|
Binary file
|
maxframe/_utils.pyi
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# Copyright 1999-2025 Alibaba Group Holding Ltd.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
from typing import Callable, Optional, Type
|
|
16
|
+
|
|
17
|
+
def to_str(s, encoding: Optional[str] = None) -> str: ...
|
|
18
|
+
def to_binary(s, encoding: Optional[str] = None) -> bytes: ...
|
|
19
|
+
def register_tokenizer(cls: Type, handler: Callable) -> None: ...
|
|
20
|
+
def reset_id_random_seed() -> None: ...
|
|
21
|
+
def new_random_id(byte_len: int) -> bytes: ...
|
maxframe/_utils.pyx
CHANGED
|
@@ -22,7 +22,6 @@ import pickle
|
|
|
22
22
|
import pkgutil
|
|
23
23
|
import time
|
|
24
24
|
import types
|
|
25
|
-
import uuid
|
|
26
25
|
import warnings
|
|
27
26
|
from datetime import date, datetime, timedelta, tzinfo
|
|
28
27
|
from enum import Enum
|
|
@@ -283,8 +282,8 @@ cdef inline tuple tokenize_numpy(ob):
|
|
|
283
282
|
try:
|
|
284
283
|
data = mmh_hash_bytes(pickle.dumps(ob, pickle.HIGHEST_PROTOCOL))
|
|
285
284
|
except:
|
|
286
|
-
# nothing can do, generate
|
|
287
|
-
data =
|
|
285
|
+
# nothing can do, generate random id
|
|
286
|
+
data = new_random_id(32).hex().encode()
|
|
288
287
|
else:
|
|
289
288
|
try:
|
|
290
289
|
data = mmh_hash_bytes(ob.ravel().view('i1').data)
|
|
@@ -397,12 +396,14 @@ def tokenize_pickled_with_cache(ob):
|
|
|
397
396
|
|
|
398
397
|
def tokenize_cupy(ob):
|
|
399
398
|
from .serialization import serialize
|
|
399
|
+
|
|
400
400
|
header, _buffers = serialize(ob)
|
|
401
401
|
return iterative_tokenize([header, ob.data.ptr])
|
|
402
402
|
|
|
403
403
|
|
|
404
404
|
def tokenize_cudf(ob):
|
|
405
405
|
from .serialization import serialize
|
|
406
|
+
|
|
406
407
|
header, buffers = serialize(ob)
|
|
407
408
|
return iterative_tokenize([header] + [(buf.ptr, buf.size) for buf in buffers])
|
|
408
409
|
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# Copyright 1999-2025 Alibaba Group Holding Ltd.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
from .core import (
|
|
16
|
+
BUILTIN_ENGINE_DPE,
|
|
17
|
+
BUILTIN_ENGINE_MCSQL,
|
|
18
|
+
BUILTIN_ENGINE_SPE,
|
|
19
|
+
AbstractUDF,
|
|
20
|
+
CodeGenResult,
|
|
21
|
+
DAGCodeContext,
|
|
22
|
+
DAGCodeGenerator,
|
|
23
|
+
DagOperatorAdapter,
|
|
24
|
+
EngineAcceptance,
|
|
25
|
+
UserCodeMixin,
|
|
26
|
+
register_engine_codegen,
|
|
27
|
+
)
|
|
@@ -23,17 +23,17 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union
|
|
|
23
23
|
from odps.types import OdpsSchema
|
|
24
24
|
from odps.utils import camel_to_underline
|
|
25
25
|
|
|
26
|
-
from
|
|
27
|
-
from
|
|
28
|
-
from
|
|
29
|
-
from
|
|
30
|
-
from
|
|
31
|
-
from
|
|
32
|
-
from
|
|
33
|
-
from
|
|
34
|
-
from
|
|
35
|
-
from
|
|
36
|
-
from
|
|
26
|
+
from ..core import OperatorType, Tileable, TileableGraph
|
|
27
|
+
from ..core.operator import Fetch, Operator
|
|
28
|
+
from ..extension import iter_extensions
|
|
29
|
+
from ..io.odpsio import build_dataframe_table_meta
|
|
30
|
+
from ..io.odpsio.schema import pandas_to_odps_schema
|
|
31
|
+
from ..lib import wrapped_pickle as pickle
|
|
32
|
+
from ..protocol import DataFrameTableMeta, ResultInfo
|
|
33
|
+
from ..serialization import PickleContainer
|
|
34
|
+
from ..serialization.serializables import Serializable, StringField
|
|
35
|
+
from ..typing_ import PandasObjectTypes
|
|
36
|
+
from ..udf import MarkedFunction, PythonPackOptions
|
|
37
37
|
|
|
38
38
|
if TYPE_CHECKING:
|
|
39
39
|
from odpsctx import ODPSSessionContext
|
|
@@ -129,7 +129,8 @@ class UserCodeMixin:
|
|
|
129
129
|
return f"{{{', '.join(items)}}}"
|
|
130
130
|
|
|
131
131
|
if isinstance(obj, tuple):
|
|
132
|
-
|
|
132
|
+
obj_exprs = [cls.obj_to_python_expr(sub_obj) for sub_obj in obj]
|
|
133
|
+
return f"({', '.join(obj_exprs)}{',' if len(obj) == 1 else ''})"
|
|
133
134
|
|
|
134
135
|
if isinstance(obj, set):
|
|
135
136
|
return (
|
|
@@ -162,10 +163,14 @@ class UserCodeMixin:
|
|
|
162
163
|
Returns
|
|
163
164
|
-------
|
|
164
165
|
str :
|
|
165
|
-
The code snippets of pickling, the final variable is called
|
|
166
|
+
The code snippets of pickling, the final variable is called
|
|
167
|
+
'pickled_data' by default.
|
|
166
168
|
"""
|
|
167
169
|
pickled, buffers = cls.dump_pickled_data(code_to_pickle)
|
|
168
|
-
pickle_loads_expr =
|
|
170
|
+
pickle_loads_expr = (
|
|
171
|
+
f"cloudpickle.loads({cls.obj_to_python_expr(pickled)}, "
|
|
172
|
+
f"buffers={cls.obj_to_python_expr(buffers)})"
|
|
173
|
+
)
|
|
169
174
|
if unpicked_data_var_name:
|
|
170
175
|
return f"{unpicked_data_var_name} = {pickle_loads_expr}"
|
|
171
176
|
|
|
@@ -187,7 +192,7 @@ class UserCodeMixin:
|
|
|
187
192
|
return pickled, buffers
|
|
188
193
|
|
|
189
194
|
|
|
190
|
-
class
|
|
195
|
+
class DAGCodeContext(metaclass=abc.ABCMeta):
|
|
191
196
|
def __init__(self, session_id: str = None, subdag_id: str = None):
|
|
192
197
|
self._session_id = session_id
|
|
193
198
|
self._subdag_id = subdag_id
|
|
@@ -204,6 +209,10 @@ class BigDagCodeContext(metaclass=abc.ABCMeta):
|
|
|
204
209
|
def session_id(self) -> str:
|
|
205
210
|
return self._session_id
|
|
206
211
|
|
|
212
|
+
@property
|
|
213
|
+
def subdag_id(self) -> str:
|
|
214
|
+
return self._subdag_id
|
|
215
|
+
|
|
207
216
|
def register_udf(self, udf: AbstractUDF):
|
|
208
217
|
udf.session_id = self._session_id
|
|
209
218
|
self._udfs[udf.name] = udf
|
|
@@ -293,7 +302,10 @@ class EngineAcceptance(Enum):
|
|
|
293
302
|
"""
|
|
294
303
|
DENY: The operator is not accepted by the current engine.
|
|
295
304
|
ACCEPT: The operator is accepted by the current engine, and doesn't break from here.
|
|
296
|
-
|
|
305
|
+
BREAK_BEFORE: The operator is accepted by the current engine, but should break before
|
|
306
|
+
its execution.
|
|
307
|
+
BREAK_AFTER: The operator is accepted by the current engine, but should break after
|
|
308
|
+
its execution.
|
|
297
309
|
PREDECESSOR: The acceptance of the operator is decided by engines of its
|
|
298
310
|
predecessors. If acceptance of all predecessors are SUCCESSOR, the acceptance
|
|
299
311
|
of current operator is SUCCESSOR. Otherwise the engine selected in predecessors
|
|
@@ -305,9 +317,10 @@ class EngineAcceptance(Enum):
|
|
|
305
317
|
|
|
306
318
|
DENY = 0
|
|
307
319
|
ACCEPT = 1
|
|
308
|
-
|
|
320
|
+
BREAK_AFTER = 2
|
|
309
321
|
PREDECESSOR = 3
|
|
310
322
|
SUCCESSOR = 4
|
|
323
|
+
BREAK_BEFORE = 5
|
|
311
324
|
|
|
312
325
|
@classmethod
|
|
313
326
|
def _missing_(cls, pred: bool) -> "EngineAcceptance":
|
|
@@ -327,18 +340,16 @@ class EngineAcceptance(Enum):
|
|
|
327
340
|
return cls.ACCEPT if pred else cls.DENY
|
|
328
341
|
|
|
329
342
|
|
|
330
|
-
class
|
|
343
|
+
class DagOperatorAdapter(metaclass=abc.ABCMeta):
|
|
331
344
|
# todo handle refcount issue when generated code is being executed
|
|
332
345
|
def accepts(self, op: Operator) -> EngineAcceptance:
|
|
333
346
|
return EngineAcceptance.ACCEPT
|
|
334
347
|
|
|
335
348
|
@abc.abstractmethod
|
|
336
|
-
def generate_code(self, op:
|
|
349
|
+
def generate_code(self, op: Operator, context: DAGCodeContext) -> List[str]:
|
|
337
350
|
raise NotImplementedError
|
|
338
351
|
|
|
339
|
-
def generate_comment(
|
|
340
|
-
self, op: OperatorType, context: BigDagCodeContext
|
|
341
|
-
) -> List[str]:
|
|
352
|
+
def generate_comment(self, op: Operator, context: DAGCodeContext) -> List[str]:
|
|
342
353
|
"""
|
|
343
354
|
Generate the comment codes before actual ones.
|
|
344
355
|
|
|
@@ -346,8 +357,8 @@ class BigDagOperatorAdapter(metaclass=abc.ABCMeta):
|
|
|
346
357
|
----------
|
|
347
358
|
op : Operator
|
|
348
359
|
The operator instance.
|
|
349
|
-
context :
|
|
350
|
-
The
|
|
360
|
+
context : DAGCodeContext
|
|
361
|
+
The DagCodeContext instance.
|
|
351
362
|
|
|
352
363
|
Returns
|
|
353
364
|
-------
|
|
@@ -356,9 +367,7 @@ class BigDagOperatorAdapter(metaclass=abc.ABCMeta):
|
|
|
356
367
|
"""
|
|
357
368
|
return list()
|
|
358
369
|
|
|
359
|
-
def generate_pre_op_code(
|
|
360
|
-
self, op: Operator, context: BigDagCodeContext
|
|
361
|
-
) -> List[str]:
|
|
370
|
+
def generate_pre_op_code(self, op: Operator, context: DAGCodeContext) -> List[str]:
|
|
362
371
|
"""
|
|
363
372
|
Generate the codes before actually handling the operator.
|
|
364
373
|
This method is usually implemented in the base class of each engine.
|
|
@@ -367,8 +376,8 @@ class BigDagOperatorAdapter(metaclass=abc.ABCMeta):
|
|
|
367
376
|
----------
|
|
368
377
|
op : Operator
|
|
369
378
|
The operator instance.
|
|
370
|
-
context :
|
|
371
|
-
The
|
|
379
|
+
context : DAGCodeContext
|
|
380
|
+
The DagCodeContext instance.
|
|
372
381
|
|
|
373
382
|
Returns
|
|
374
383
|
-------
|
|
@@ -377,9 +386,7 @@ class BigDagOperatorAdapter(metaclass=abc.ABCMeta):
|
|
|
377
386
|
"""
|
|
378
387
|
return list()
|
|
379
388
|
|
|
380
|
-
def generate_post_op_code(
|
|
381
|
-
self, op: Operator, context: BigDagCodeContext
|
|
382
|
-
) -> List[str]:
|
|
389
|
+
def generate_post_op_code(self, op: Operator, context: DAGCodeContext) -> List[str]:
|
|
383
390
|
"""
|
|
384
391
|
Generate the codes after actually handling the operator.
|
|
385
392
|
This method is usually implemented in the base class of each engine.
|
|
@@ -388,8 +395,8 @@ class BigDagOperatorAdapter(metaclass=abc.ABCMeta):
|
|
|
388
395
|
----------
|
|
389
396
|
op : Operator
|
|
390
397
|
The operator instance.
|
|
391
|
-
context :
|
|
392
|
-
The
|
|
398
|
+
context : DAGCodeContext
|
|
399
|
+
The DagCodeContext instance.
|
|
393
400
|
|
|
394
401
|
Returns
|
|
395
402
|
-------
|
|
@@ -399,20 +406,21 @@ class BigDagOperatorAdapter(metaclass=abc.ABCMeta):
|
|
|
399
406
|
return list()
|
|
400
407
|
|
|
401
408
|
|
|
402
|
-
_engine_to_codegen: Dict[str, Type["
|
|
409
|
+
_engine_to_codegen: Dict[str, Type["DAGCodeGenerator"]] = dict()
|
|
403
410
|
|
|
404
411
|
|
|
405
|
-
def register_engine_codegen(type_: Type["
|
|
412
|
+
def register_engine_codegen(type_: Type["DAGCodeGenerator"]):
|
|
406
413
|
_engine_to_codegen[type_.engine_type] = type_
|
|
407
414
|
return type_
|
|
408
415
|
|
|
409
416
|
|
|
417
|
+
BUILTIN_ENGINE_DPE = "DPE"
|
|
410
418
|
BUILTIN_ENGINE_SPE = "SPE"
|
|
411
419
|
BUILTIN_ENGINE_MCSQL = "MCSQL"
|
|
412
420
|
|
|
413
421
|
|
|
414
|
-
class
|
|
415
|
-
_context:
|
|
422
|
+
class DAGCodeGenerator(metaclass=abc.ABCMeta):
|
|
423
|
+
_context: DAGCodeContext
|
|
416
424
|
|
|
417
425
|
engine_type: Optional[str] = None
|
|
418
426
|
engine_priority: int = 0
|
|
@@ -442,18 +450,16 @@ class BigDagCodeGenerator(metaclass=abc.ABCMeta):
|
|
|
442
450
|
return [e.engine_type for e in engines]
|
|
443
451
|
|
|
444
452
|
@classmethod
|
|
445
|
-
def get_by_engine_type(cls, engine_type: str) -> Type["
|
|
453
|
+
def get_by_engine_type(cls, engine_type: str) -> Type["DAGCodeGenerator"]:
|
|
446
454
|
cls._load_engine_extensions()
|
|
447
455
|
return _engine_to_codegen[engine_type]
|
|
448
456
|
|
|
449
457
|
@abc.abstractmethod
|
|
450
|
-
def get_op_adapter(
|
|
451
|
-
self, op_type: Type[OperatorType]
|
|
452
|
-
) -> Type[BigDagOperatorAdapter]:
|
|
458
|
+
def get_op_adapter(self, op_type: Type[OperatorType]) -> Type[DagOperatorAdapter]:
|
|
453
459
|
raise NotImplementedError
|
|
454
460
|
|
|
455
461
|
@abc.abstractmethod
|
|
456
|
-
def _init_context(self, session_id: str, subdag_id: str) ->
|
|
462
|
+
def _init_context(self, session_id: str, subdag_id: str) -> DAGCodeContext:
|
|
457
463
|
raise NotImplementedError
|
|
458
464
|
|
|
459
465
|
def _generate_delete_code(self, var_name: str) -> List[str]:
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# Copyright 1999-2025 Alibaba Group Holding Ltd.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
from . import dataframe, learn, objects, remote, tensor
|
|
16
|
+
from .core import SPECodeGenerator
|
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
# Copyright 1999-2025 Alibaba Group Holding Ltd.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
import itertools
|
|
16
|
+
from abc import ABC, abstractmethod
|
|
17
|
+
from collections import defaultdict
|
|
18
|
+
from typing import Any, Callable, Dict, Iterable, List, Optional, Type, Union
|
|
19
|
+
|
|
20
|
+
import pandas as pd
|
|
21
|
+
from numpy import dtype as npdtype
|
|
22
|
+
from numpy import ndarray
|
|
23
|
+
from pandas.tseries.offsets import BaseOffset
|
|
24
|
+
|
|
25
|
+
from ...core import TILEABLE_TYPE, OperatorType, TileableGraph
|
|
26
|
+
from ...core.operator.base import Operator
|
|
27
|
+
from ...serialization import PickleContainer
|
|
28
|
+
from ...utils import TypeDispatcher, no_default
|
|
29
|
+
from ..core import (
|
|
30
|
+
BUILTIN_ENGINE_SPE,
|
|
31
|
+
DAGCodeContext,
|
|
32
|
+
DAGCodeGenerator,
|
|
33
|
+
DagOperatorAdapter,
|
|
34
|
+
register_engine_codegen,
|
|
35
|
+
)
|
|
36
|
+
from .dataframe.udf import SpeUDF
|
|
37
|
+
|
|
38
|
+
_spe_op_adapter = TypeDispatcher()
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def register_op_adapter(op_cls: Union[Type[OperatorType], List[Type[OperatorType]]]):
|
|
42
|
+
def wrapper(cls: Type[DagOperatorAdapter]):
|
|
43
|
+
op_classes = op_cls
|
|
44
|
+
if not isinstance(op_cls, list):
|
|
45
|
+
op_classes = [op_cls]
|
|
46
|
+
for op_class in op_classes:
|
|
47
|
+
_spe_op_adapter.register(op_class, cls)
|
|
48
|
+
return cls
|
|
49
|
+
|
|
50
|
+
return wrapper
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class SPECodeContext(DAGCodeContext, ABC):
|
|
54
|
+
logger_var: str = "logger"
|
|
55
|
+
|
|
56
|
+
def __init__(self, session_id: str = None, subdag_id: str = None):
|
|
57
|
+
super().__init__(session_id, subdag_id)
|
|
58
|
+
self.imports = defaultdict(set)
|
|
59
|
+
self.import_froms = defaultdict(set)
|
|
60
|
+
|
|
61
|
+
def register_import(
|
|
62
|
+
self, mod: str, alias: Optional[str] = None, from_item: Optional[str] = None
|
|
63
|
+
):
|
|
64
|
+
if from_item is None:
|
|
65
|
+
self.imports[mod].add(alias)
|
|
66
|
+
else:
|
|
67
|
+
self.import_froms[mod].add((from_item, alias))
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class SPEOperatorAdapter(DagOperatorAdapter, ABC):
|
|
71
|
+
@abstractmethod
|
|
72
|
+
def generate_code(self, op: OperatorType, context: SPECodeContext) -> List[str]:
|
|
73
|
+
raise NotImplementedError
|
|
74
|
+
|
|
75
|
+
@classmethod
|
|
76
|
+
def generate_call_args_with_attributes(
|
|
77
|
+
cls,
|
|
78
|
+
op: OperatorType,
|
|
79
|
+
context: SPECodeContext,
|
|
80
|
+
*args: str,
|
|
81
|
+
skip_none: bool = False,
|
|
82
|
+
kw_keys: Optional[Iterable[str]] = None,
|
|
83
|
+
**kwargs: Optional[str],
|
|
84
|
+
) -> str:
|
|
85
|
+
"""
|
|
86
|
+
Generate the codes with simple copying attributes from the input operator to the
|
|
87
|
+
output Python call parameters. The absence of the attribute value will be
|
|
88
|
+
fallback to None.
|
|
89
|
+
|
|
90
|
+
Parameters
|
|
91
|
+
----------
|
|
92
|
+
op: OperatorType
|
|
93
|
+
The operator instance.
|
|
94
|
+
context: SPECodeContext
|
|
95
|
+
The SPECodeContext instance.
|
|
96
|
+
*args:
|
|
97
|
+
The attribute names, which will remain the same name in the call parameter.
|
|
98
|
+
skip_none: bool
|
|
99
|
+
If True, will skip attributes with None value.
|
|
100
|
+
**kwargs:
|
|
101
|
+
The keyword arguments with string values, which will map to the name of the
|
|
102
|
+
call parameter.
|
|
103
|
+
|
|
104
|
+
e.g. old_name="new_name" will be __call__(new_name=getattr(obj, old_name))
|
|
105
|
+
|
|
106
|
+
Returns
|
|
107
|
+
-------
|
|
108
|
+
str :
|
|
109
|
+
The call args codes.
|
|
110
|
+
"""
|
|
111
|
+
kw = {key: kwargs.get(key) for key in kw_keys or ()}
|
|
112
|
+
kw.update(kwargs)
|
|
113
|
+
str_args = [a for a in args if isinstance(a, str)]
|
|
114
|
+
all_args_dict = cls._collect_op_kwargs(
|
|
115
|
+
op, str_args + list(kw.keys()), skip_none=skip_none
|
|
116
|
+
)
|
|
117
|
+
args_list = [
|
|
118
|
+
all_args_dict.get(k) if isinstance(k, str) else op.inputs[k] for k in args
|
|
119
|
+
]
|
|
120
|
+
kwargs_dict = {
|
|
121
|
+
kw.get(k) or k: all_args_dict.get(k)
|
|
122
|
+
for k in kw.keys()
|
|
123
|
+
if k in all_args_dict
|
|
124
|
+
}
|
|
125
|
+
return ", ".join(cls._translate_call_args(context, *args_list, **kwargs_dict))
|
|
126
|
+
|
|
127
|
+
@classmethod
|
|
128
|
+
def _collect_op_kwargs(
|
|
129
|
+
cls, op: OperatorType, args: List[str], skip_none: bool = True
|
|
130
|
+
) -> Dict[str, str]:
|
|
131
|
+
kw = {}
|
|
132
|
+
for arg in args:
|
|
133
|
+
attr_val = getattr(op, arg, None)
|
|
134
|
+
if attr_val is not no_default and (not skip_none or attr_val is not None):
|
|
135
|
+
kw[arg] = attr_val
|
|
136
|
+
return kw
|
|
137
|
+
|
|
138
|
+
@classmethod
|
|
139
|
+
def _translate_call_args(
|
|
140
|
+
cls, context: SPECodeContext, *args, **kwargs
|
|
141
|
+
) -> List[str]:
|
|
142
|
+
return [cls.translate_var(context, v) for v in args] + [
|
|
143
|
+
f"{k}={cls.translate_var(context, v)}" for k, v in kwargs.items()
|
|
144
|
+
]
|
|
145
|
+
|
|
146
|
+
@classmethod
|
|
147
|
+
def translate_var(cls, context: SPECodeContext, val: Any) -> str:
|
|
148
|
+
"""
|
|
149
|
+
Translate the val to a Python variable.
|
|
150
|
+
|
|
151
|
+
Parameters
|
|
152
|
+
----------
|
|
153
|
+
context: SPECodeContext
|
|
154
|
+
The SPECodeContext instance.
|
|
155
|
+
val : Any
|
|
156
|
+
The value to be translated.
|
|
157
|
+
|
|
158
|
+
Returns
|
|
159
|
+
-------
|
|
160
|
+
str :
|
|
161
|
+
The var name which can be used in generated Python code.
|
|
162
|
+
"""
|
|
163
|
+
if isinstance(val, TILEABLE_TYPE):
|
|
164
|
+
return context.get_input_tileable_variable(val)
|
|
165
|
+
|
|
166
|
+
if isinstance(val, (Callable, PickleContainer)) and not isinstance(
|
|
167
|
+
val, BaseOffset
|
|
168
|
+
):
|
|
169
|
+
# TODO: handle used resources here
|
|
170
|
+
context.register_import("base64")
|
|
171
|
+
context.register_import("cloudpickle")
|
|
172
|
+
context.register_import("numpy", "np")
|
|
173
|
+
udf = SpeUDF(val)
|
|
174
|
+
context.register_udf(udf)
|
|
175
|
+
return udf.name
|
|
176
|
+
|
|
177
|
+
val_type = type(val)
|
|
178
|
+
if val_type is list:
|
|
179
|
+
vals_exp = [cls.translate_var(context, v) for v in val]
|
|
180
|
+
return f"[{', '.join(vals_exp)}]"
|
|
181
|
+
|
|
182
|
+
if val_type is tuple:
|
|
183
|
+
vals_exp = [cls.translate_var(context, v) for v in val]
|
|
184
|
+
if len(vals_exp) == 1:
|
|
185
|
+
# need a trailing space to make a tuple
|
|
186
|
+
vals_exp.append("")
|
|
187
|
+
return f"({', '.join(vals_exp).rstrip()})"
|
|
188
|
+
|
|
189
|
+
if val_type is slice:
|
|
190
|
+
return f"slice({val.start}, {val.stop}, {val.step})"
|
|
191
|
+
|
|
192
|
+
if val_type is ndarray:
|
|
193
|
+
return cls.translate_var(context, val.tolist())
|
|
194
|
+
|
|
195
|
+
if val_type is dict:
|
|
196
|
+
kvs = list()
|
|
197
|
+
for k, v in val.items():
|
|
198
|
+
kvs.append(
|
|
199
|
+
f"{cls.translate_var(context, k)}: {cls.translate_var(context, v)}"
|
|
200
|
+
)
|
|
201
|
+
return f"{{{', '.join(kvs)}}}"
|
|
202
|
+
|
|
203
|
+
if val_type is set:
|
|
204
|
+
keys = [cls.translate_var(context, k) for k in val]
|
|
205
|
+
return f"{{{', '.join(keys)}}}"
|
|
206
|
+
|
|
207
|
+
if val_type is pd.Timestamp:
|
|
208
|
+
context.register_import("pandas", "pd")
|
|
209
|
+
return f"pd.Timestamp({str(val)!r})"
|
|
210
|
+
|
|
211
|
+
if isinstance(val, npdtype):
|
|
212
|
+
context.register_import("numpy", "np")
|
|
213
|
+
return f"np.dtype({str(val)!r})"
|
|
214
|
+
|
|
215
|
+
return context.register_operator_constants(val)
|
|
216
|
+
|
|
217
|
+
def generate_pre_op_code(self, op: Operator, context: SPECodeContext) -> List[str]:
|
|
218
|
+
context.register_operator_constants(True, "running")
|
|
219
|
+
return ["if not running:\n raise RuntimeError('CANCELLED')"]
|
|
220
|
+
|
|
221
|
+
def gen_logging_code(
|
|
222
|
+
self, context: SPECodeContext, message: str, expressions: List[str]
|
|
223
|
+
) -> str:
|
|
224
|
+
exp_str = ", ".join(expressions)
|
|
225
|
+
return f"{context.logger_var}.info('{message}', {exp_str})"
|
|
226
|
+
|
|
227
|
+
def gen_timecost_code(
|
|
228
|
+
self, context: SPECodeContext, phase_name: str, code_lines: List[str]
|
|
229
|
+
) -> List[str]:
|
|
230
|
+
context.register_import("time")
|
|
231
|
+
codes = ["start_time = time.time()"]
|
|
232
|
+
codes.extend(code_lines)
|
|
233
|
+
codes.append(
|
|
234
|
+
f"{context.logger_var}.info('{phase_name} cost: %.2f s', time.time() - start_time)"
|
|
235
|
+
)
|
|
236
|
+
return codes
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
@register_engine_codegen
|
|
240
|
+
class SPECodeGenerator(DAGCodeGenerator):
|
|
241
|
+
_context: SPECodeContext
|
|
242
|
+
|
|
243
|
+
engine_type = BUILTIN_ENGINE_SPE
|
|
244
|
+
engine_priority = 0
|
|
245
|
+
|
|
246
|
+
def _init_context(self, session_id: str, subdag_id: str) -> SPECodeContext:
|
|
247
|
+
return SPECodeContext(session_id, subdag_id)
|
|
248
|
+
|
|
249
|
+
def _generate_delete_code(self, var_name: str) -> List[str]:
|
|
250
|
+
return [f"del {var_name}"]
|
|
251
|
+
|
|
252
|
+
@staticmethod
|
|
253
|
+
def _generate_import_code(module: str, alias: Optional[str] = None):
|
|
254
|
+
if alias is None:
|
|
255
|
+
return f"import {module}"
|
|
256
|
+
else:
|
|
257
|
+
return f"import {module} as {alias}"
|
|
258
|
+
|
|
259
|
+
@staticmethod
|
|
260
|
+
def _generate_import_from_code(
|
|
261
|
+
module: str, from_list: List[str], alias_list: Optional[List[str]] = None
|
|
262
|
+
):
|
|
263
|
+
def build_from_str(from_str: str, alias: Optional[str]) -> str:
|
|
264
|
+
if alias is not None:
|
|
265
|
+
return f"{from_str} as {alias}"
|
|
266
|
+
else:
|
|
267
|
+
return from_str
|
|
268
|
+
|
|
269
|
+
alias_list = alias_list or itertools.repeat(None)
|
|
270
|
+
froms_str = ", ".join(
|
|
271
|
+
build_from_str(f, a) for f, a in zip(from_list, alias_list)
|
|
272
|
+
)
|
|
273
|
+
return f"from {module} import {froms_str}"
|
|
274
|
+
|
|
275
|
+
def get_op_adapter(self, op_type: Type[OperatorType]) -> Type[DagOperatorAdapter]:
|
|
276
|
+
return get_op_adapter(op_type)
|
|
277
|
+
|
|
278
|
+
def generate_code(self, dag: TileableGraph) -> List[str]:
|
|
279
|
+
from . import dataframe, tensor
|
|
280
|
+
|
|
281
|
+
del dataframe, tensor
|
|
282
|
+
main_codes = super().generate_code(dag)
|
|
283
|
+
import_codes = []
|
|
284
|
+
for mod, aliases in self._context.imports.items():
|
|
285
|
+
for alias in aliases:
|
|
286
|
+
import_codes.append(self._generate_import_code(mod, alias))
|
|
287
|
+
for mod, from_tuples in self._context.import_froms.items():
|
|
288
|
+
from_mods, aliases = [], []
|
|
289
|
+
for from_item, from_alias in from_tuples:
|
|
290
|
+
from_mods.append(from_item)
|
|
291
|
+
aliases.append(from_alias)
|
|
292
|
+
import_codes.append(
|
|
293
|
+
self._generate_import_from_code(mod, from_mods, aliases)
|
|
294
|
+
)
|
|
295
|
+
udf_codes = self._generate_udf_codes()
|
|
296
|
+
return import_codes + udf_codes + main_codes
|
|
297
|
+
|
|
298
|
+
def _generate_udf_codes(self) -> List[str]:
|
|
299
|
+
udf_codes = list()
|
|
300
|
+
for func in self.get_udfs():
|
|
301
|
+
udf_codes.extend(func.encoded_content)
|
|
302
|
+
udf_codes.append(f"{func.name} = pickled_data")
|
|
303
|
+
return udf_codes
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def get_op_adapter(op_type: Type[OperatorType]) -> Type[SPEOperatorAdapter]:
|
|
307
|
+
return _spe_op_adapter.get_handler(op_type)
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# Copyright 1999-2025 Alibaba Group Holding Ltd.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
from . import (
|
|
16
|
+
accessors,
|
|
17
|
+
arithmetic,
|
|
18
|
+
datasource,
|
|
19
|
+
datastore,
|
|
20
|
+
fetch,
|
|
21
|
+
groupby,
|
|
22
|
+
indexing,
|
|
23
|
+
merge,
|
|
24
|
+
misc,
|
|
25
|
+
missing,
|
|
26
|
+
reduction,
|
|
27
|
+
sort,
|
|
28
|
+
statistics,
|
|
29
|
+
tseries,
|
|
30
|
+
value_counts,
|
|
31
|
+
window,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
try:
|
|
35
|
+
from . import _internal
|
|
36
|
+
except ImportError:
|
|
37
|
+
pass
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Copyright 1999-2025 Alibaba Group Holding Ltd.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
from . import base, dict_, list_
|