snowpark-connect 0.24.0__py3-none-any.whl → 0.25.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of snowpark-connect might be problematic. Click here for more details.
- snowflake/snowpark_connect/column_name_handler.py +116 -4
- snowflake/snowpark_connect/config.py +13 -0
- snowflake/snowpark_connect/constants.py +0 -29
- snowflake/snowpark_connect/dataframe_container.py +6 -0
- snowflake/snowpark_connect/execute_plan/map_execution_command.py +56 -1
- snowflake/snowpark_connect/expression/literal.py +13 -2
- snowflake/snowpark_connect/expression/map_cast.py +5 -8
- snowflake/snowpark_connect/expression/map_sql_expression.py +23 -1
- snowflake/snowpark_connect/expression/map_udf.py +26 -8
- snowflake/snowpark_connect/expression/map_unresolved_attribute.py +199 -15
- snowflake/snowpark_connect/expression/map_unresolved_extract_value.py +44 -16
- snowflake/snowpark_connect/expression/map_unresolved_function.py +825 -353
- snowflake/snowpark_connect/expression/map_unresolved_star.py +3 -2
- snowflake/snowpark_connect/hidden_column.py +39 -0
- snowflake/snowpark_connect/includes/jars/hadoop-client-api-trimmed-3.3.4.jar +0 -0
- snowflake/snowpark_connect/includes/jars/{hadoop-client-api-3.3.4.jar → spark-connect-client-jvm_2.12-3.5.6.jar} +0 -0
- snowflake/snowpark_connect/relation/map_column_ops.py +17 -4
- snowflake/snowpark_connect/relation/map_extension.py +52 -11
- snowflake/snowpark_connect/relation/map_join.py +258 -62
- snowflake/snowpark_connect/relation/map_sql.py +88 -11
- snowflake/snowpark_connect/relation/map_udtf.py +4 -2
- snowflake/snowpark_connect/relation/read/map_read.py +3 -3
- snowflake/snowpark_connect/relation/read/map_read_jdbc.py +1 -1
- snowflake/snowpark_connect/relation/read/map_read_json.py +8 -1
- snowflake/snowpark_connect/relation/read/map_read_table.py +1 -9
- snowflake/snowpark_connect/relation/read/reader_config.py +3 -1
- snowflake/snowpark_connect/relation/write/map_write.py +62 -53
- snowflake/snowpark_connect/resources_initializer.py +29 -1
- snowflake/snowpark_connect/server.py +18 -3
- snowflake/snowpark_connect/type_mapping.py +29 -25
- snowflake/snowpark_connect/typed_column.py +14 -0
- snowflake/snowpark_connect/utils/artifacts.py +23 -0
- snowflake/snowpark_connect/utils/context.py +6 -1
- snowflake/snowpark_connect/utils/scala_udf_utils.py +588 -0
- snowflake/snowpark_connect/utils/telemetry.py +6 -17
- snowflake/snowpark_connect/utils/udf_helper.py +2 -0
- snowflake/snowpark_connect/utils/udf_utils.py +38 -7
- snowflake/snowpark_connect/utils/udtf_utils.py +17 -3
- snowflake/snowpark_connect/version.py +1 -1
- {snowpark_connect-0.24.0.dist-info → snowpark_connect-0.25.0.dist-info}/METADATA +1 -1
- snowpark_connect-0.25.0.dist-info/RECORD +477 -0
- snowflake/snowpark_connect/includes/jars/scala-compiler-2.12.18.jar +0 -0
- snowflake/snowpark_connect/includes/jars/spark-kubernetes_2.12-3.5.6.jar +0 -0
- snowflake/snowpark_connect/includes/jars/spark-mllib_2.12-3.5.6.jar +0 -0
- snowflake/snowpark_connect/includes/jars/spark-streaming_2.12-3.5.6.jar +0 -0
- snowflake/snowpark_connect/includes/python/pyspark/errors/tests/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/errors/tests/test_errors.py +0 -60
- snowflake/snowpark_connect/includes/python/pyspark/ml/deepspeed/tests/test_deepspeed_distributor.py +0 -306
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_connect_classification.py +0 -53
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_connect_evaluation.py +0 -50
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_connect_feature.py +0 -43
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_connect_function.py +0 -114
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_connect_pipeline.py +0 -47
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_connect_summarizer.py +0 -43
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_connect_tuning.py +0 -46
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_legacy_mode_classification.py +0 -238
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_legacy_mode_evaluation.py +0 -194
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_legacy_mode_feature.py +0 -156
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_legacy_mode_pipeline.py +0 -184
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_legacy_mode_summarizer.py +0 -78
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_legacy_mode_tuning.py +0 -292
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_parity_torch_data_loader.py +0 -50
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_parity_torch_distributor.py +0 -152
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_algorithms.py +0 -456
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_base.py +0 -96
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_dl_util.py +0 -186
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_evaluation.py +0 -77
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_feature.py +0 -401
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_functions.py +0 -528
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_image.py +0 -82
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_linalg.py +0 -409
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_model_cache.py +0 -55
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_param.py +0 -441
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_persistence.py +0 -546
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_pipeline.py +0 -71
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_stat.py +0 -52
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_training_summary.py +0 -494
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_util.py +0 -85
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_wrapper.py +0 -138
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/test_cv_io_basic.py +0 -151
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/test_cv_io_nested.py +0 -97
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/test_cv_io_pipeline.py +0 -143
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/test_tuning.py +0 -551
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/test_tvs_io_basic.py +0 -137
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/test_tvs_io_nested.py +0 -96
- snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/test_tvs_io_pipeline.py +0 -142
- snowflake/snowpark_connect/includes/python/pyspark/ml/torch/tests/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/ml/torch/tests/test_data_loader.py +0 -137
- snowflake/snowpark_connect/includes/python/pyspark/ml/torch/tests/test_distributor.py +0 -561
- snowflake/snowpark_connect/includes/python/pyspark/ml/torch/tests/test_log_communication.py +0 -172
- snowflake/snowpark_connect/includes/python/pyspark/mllib/tests/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/mllib/tests/test_algorithms.py +0 -353
- snowflake/snowpark_connect/includes/python/pyspark/mllib/tests/test_feature.py +0 -192
- snowflake/snowpark_connect/includes/python/pyspark/mllib/tests/test_linalg.py +0 -680
- snowflake/snowpark_connect/includes/python/pyspark/mllib/tests/test_stat.py +0 -206
- snowflake/snowpark_connect/includes/python/pyspark/mllib/tests/test_streaming_algorithms.py +0 -471
- snowflake/snowpark_connect/includes/python/pyspark/mllib/tests/test_util.py +0 -108
- snowflake/snowpark_connect/includes/python/pyspark/pandas/spark/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/pandas/spark/accessors.py +0 -1281
- snowflake/snowpark_connect/includes/python/pyspark/pandas/spark/functions.py +0 -203
- snowflake/snowpark_connect/includes/python/pyspark/pandas/spark/utils.py +0 -202
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_any_all.py +0 -177
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_apply_func.py +0 -575
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_binary_ops.py +0 -235
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_combine.py +0 -653
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_compute.py +0 -463
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_corrwith.py +0 -86
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_cov.py +0 -151
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_cumulative.py +0 -139
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_describe.py +0 -458
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_eval.py +0 -86
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_melt.py +0 -202
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_missing_data.py +0 -520
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_pivot.py +0 -361
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_any_all.py +0 -40
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_apply_func.py +0 -42
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_binary_ops.py +0 -40
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_combine.py +0 -37
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_compute.py +0 -60
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_corrwith.py +0 -40
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_cov.py +0 -40
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_cumulative.py +0 -90
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_describe.py +0 -40
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_eval.py +0 -40
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_melt.py +0 -40
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_missing_data.py +0 -42
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_pivot.py +0 -37
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_base.py +0 -36
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_binary_ops.py +0 -42
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_boolean_ops.py +0 -47
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_categorical_ops.py +0 -55
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_complex_ops.py +0 -40
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_date_ops.py +0 -47
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_datetime_ops.py +0 -47
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_null_ops.py +0 -42
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_num_arithmetic.py +0 -43
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_num_ops.py +0 -47
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_num_reverse.py +0 -43
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_string_ops.py +0 -47
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_timedelta_ops.py +0 -47
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_udt_ops.py +0 -40
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/testing_utils.py +0 -226
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_align.py +0 -39
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_basic_slow.py +0 -55
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_cov_corrwith.py +0 -39
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_dot_frame.py +0 -39
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_dot_series.py +0 -39
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_index.py +0 -39
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_series.py +0 -39
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_setitem_frame.py +0 -43
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_setitem_series.py +0 -43
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_attrs.py +0 -40
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_constructor.py +0 -39
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_conversion.py +0 -42
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_reindexing.py +0 -42
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_reshaping.py +0 -37
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_spark.py +0 -40
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_take.py +0 -42
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_time_series.py +0 -48
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_truncate.py +0 -40
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_aggregate.py +0 -40
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_apply_func.py +0 -41
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_cumulative.py +0 -67
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_describe.py +0 -40
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_groupby.py +0 -55
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_head_tail.py +0 -40
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_index.py +0 -38
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_missing_data.py +0 -55
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_split_apply.py +0 -39
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_stat.py +0 -38
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_align.py +0 -40
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_base.py +0 -50
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_category.py +0 -73
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_datetime.py +0 -39
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_indexing.py +0 -40
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_reindex.py +0 -40
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_rename.py +0 -40
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_reset_index.py +0 -48
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_timedelta.py +0 -39
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/io/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/io/test_parity_io.py +0 -40
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/plot/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot.py +0 -45
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_matplotlib.py +0 -45
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_plotly.py +0 -49
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot.py +0 -37
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_matplotlib.py +0 -53
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_plotly.py +0 -45
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_all_any.py +0 -38
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_arg_ops.py +0 -37
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_as_of.py +0 -37
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_as_type.py +0 -38
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_compute.py +0 -37
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_conversion.py +0 -40
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_cumulative.py +0 -40
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_index.py +0 -38
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_missing_data.py +0 -40
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_series.py +0 -37
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_sort.py +0 -38
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_stat.py +0 -38
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_categorical.py +0 -66
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_config.py +0 -37
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_csv.py +0 -37
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_dataframe_conversion.py +0 -42
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_dataframe_spark_io.py +0 -39
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_default_index.py +0 -49
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_ewm.py +0 -37
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_expanding.py +0 -39
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_extension.py +0 -49
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_frame_spark.py +0 -53
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_generic_functions.py +0 -43
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_indexing.py +0 -49
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_indexops_spark.py +0 -39
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_internal.py +0 -41
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_namespace.py +0 -39
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_numpy_compat.py +0 -60
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_ops_on_diff_frames.py +0 -48
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_ops_on_diff_frames_groupby.py +0 -39
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_ops_on_diff_frames_groupby_expanding.py +0 -44
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_ops_on_diff_frames_groupby_rolling.py +0 -84
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_repr.py +0 -37
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_resample.py +0 -45
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_reshape.py +0 -39
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_rolling.py +0 -39
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_scalars.py +0 -37
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_series_conversion.py +0 -39
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_series_datetime.py +0 -39
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_series_string.py +0 -39
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_spark_functions.py +0 -39
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_sql.py +0 -43
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_stats.py +0 -37
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_typedef.py +0 -36
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_utils.py +0 -37
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_window.py +0 -39
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_base.py +0 -107
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_binary_ops.py +0 -224
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_boolean_ops.py +0 -825
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_categorical_ops.py +0 -562
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_complex_ops.py +0 -368
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_date_ops.py +0 -257
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_datetime_ops.py +0 -260
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_null_ops.py +0 -178
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_num_arithmetic.py +0 -184
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_num_ops.py +0 -497
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_num_reverse.py +0 -140
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_string_ops.py +0 -354
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_timedelta_ops.py +0 -219
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_udt_ops.py +0 -192
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/testing_utils.py +0 -228
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_align.py +0 -118
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_basic_slow.py +0 -198
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_cov_corrwith.py +0 -181
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_dot_frame.py +0 -103
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_dot_series.py +0 -141
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_index.py +0 -109
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_series.py +0 -136
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_setitem_frame.py +0 -125
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_setitem_series.py +0 -217
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_attrs.py +0 -384
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_constructor.py +0 -598
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_conversion.py +0 -73
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_reindexing.py +0 -869
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_reshaping.py +0 -487
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_spark.py +0 -309
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_take.py +0 -156
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_time_series.py +0 -149
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_truncate.py +0 -163
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_aggregate.py +0 -311
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_apply_func.py +0 -524
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_cumulative.py +0 -419
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_describe.py +0 -144
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_groupby.py +0 -979
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_head_tail.py +0 -234
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_index.py +0 -206
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_missing_data.py +0 -421
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_split_apply.py +0 -187
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_stat.py +0 -397
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_align.py +0 -100
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_base.py +0 -2743
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_category.py +0 -484
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_datetime.py +0 -276
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_indexing.py +0 -432
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_reindex.py +0 -310
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_rename.py +0 -257
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_reset_index.py +0 -160
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_timedelta.py +0 -128
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/io/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/io/test_io.py +0 -137
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/plot/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/plot/test_frame_plot.py +0 -170
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/plot/test_frame_plot_matplotlib.py +0 -547
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/plot/test_frame_plot_plotly.py +0 -285
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/plot/test_series_plot.py +0 -106
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/plot/test_series_plot_matplotlib.py +0 -409
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/plot/test_series_plot_plotly.py +0 -247
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_all_any.py +0 -105
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_arg_ops.py +0 -197
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_as_of.py +0 -137
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_as_type.py +0 -227
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_compute.py +0 -634
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_conversion.py +0 -88
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_cumulative.py +0 -139
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_index.py +0 -475
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_missing_data.py +0 -265
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_series.py +0 -818
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_sort.py +0 -162
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_stat.py +0 -780
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_categorical.py +0 -741
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_config.py +0 -160
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_csv.py +0 -453
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_dataframe_conversion.py +0 -281
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_dataframe_spark_io.py +0 -487
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_default_index.py +0 -109
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_ewm.py +0 -434
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_expanding.py +0 -253
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_extension.py +0 -152
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_frame_spark.py +0 -162
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_generic_functions.py +0 -234
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_indexing.py +0 -1339
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_indexops_spark.py +0 -82
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_internal.py +0 -124
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_namespace.py +0 -638
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_numpy_compat.py +0 -200
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_ops_on_diff_frames.py +0 -1355
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby.py +0 -655
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_expanding.py +0 -113
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_rolling.py +0 -118
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_repr.py +0 -192
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_resample.py +0 -346
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_reshape.py +0 -495
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_rolling.py +0 -263
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_scalars.py +0 -59
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_series_conversion.py +0 -85
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_series_datetime.py +0 -364
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_series_string.py +0 -362
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_spark_functions.py +0 -46
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_sql.py +0 -123
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_stats.py +0 -581
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_typedef.py +0 -447
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_utils.py +0 -301
- snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_window.py +0 -465
- snowflake/snowpark_connect/includes/python/pyspark/resource/tests/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/resource/tests/test_resources.py +0 -83
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/client/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/client/test_artifact.py +0 -420
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/client/test_client.py +0 -358
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/streaming/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/streaming/test_parity_foreach.py +0 -36
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/streaming/test_parity_foreach_batch.py +0 -44
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/streaming/test_parity_listener.py +0 -116
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/streaming/test_parity_streaming.py +0 -35
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_connect_basic.py +0 -3612
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_connect_column.py +0 -1042
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_connect_function.py +0 -2381
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_connect_plan.py +0 -1060
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_arrow.py +0 -163
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_arrow_map.py +0 -38
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_arrow_python_udf.py +0 -48
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_catalog.py +0 -36
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_column.py +0 -55
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_conf.py +0 -36
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_dataframe.py +0 -96
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_datasources.py +0 -44
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_errors.py +0 -36
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_functions.py +0 -59
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_group.py +0 -36
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_pandas_cogrouped_map.py +0 -59
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_pandas_grouped_map.py +0 -74
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_pandas_grouped_map_with_state.py +0 -62
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_pandas_map.py +0 -58
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_pandas_udf.py +0 -70
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_pandas_udf_grouped_agg.py +0 -50
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_pandas_udf_scalar.py +0 -68
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_pandas_udf_window.py +0 -40
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_readwriter.py +0 -46
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_serde.py +0 -44
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_types.py +0 -100
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_udf.py +0 -100
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_udtf.py +0 -163
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_session.py +0 -181
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_utils.py +0 -42
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py +0 -623
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_grouped_map.py +0 -869
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_grouped_map_with_state.py +0 -342
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_map.py +0 -436
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_udf.py +0 -363
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_udf_grouped_agg.py +0 -592
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_udf_scalar.py +0 -1503
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_udf_typehints.py +0 -392
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_udf_typehints_with_future_annotations.py +0 -375
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_udf_window.py +0 -411
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/streaming/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/streaming/test_streaming.py +0 -401
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/streaming/test_streaming_foreach.py +0 -295
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/streaming/test_streaming_foreach_batch.py +0 -106
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/streaming/test_streaming_listener.py +0 -558
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_arrow.py +0 -1346
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_arrow_map.py +0 -182
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_arrow_python_udf.py +0 -202
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_catalog.py +0 -503
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_column.py +0 -225
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_conf.py +0 -83
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_context.py +0 -201
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_dataframe.py +0 -1931
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_datasources.py +0 -256
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_errors.py +0 -69
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_functions.py +0 -1349
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_group.py +0 -53
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_pandas_sqlmetrics.py +0 -68
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_readwriter.py +0 -283
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_serde.py +0 -155
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_session.py +0 -412
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_types.py +0 -1581
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_udf.py +0 -961
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_udf_profiler.py +0 -165
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_udtf.py +0 -1456
- snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_utils.py +0 -1686
- snowflake/snowpark_connect/includes/python/pyspark/streaming/tests/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/streaming/tests/test_context.py +0 -184
- snowflake/snowpark_connect/includes/python/pyspark/streaming/tests/test_dstream.py +0 -706
- snowflake/snowpark_connect/includes/python/pyspark/streaming/tests/test_kinesis.py +0 -118
- snowflake/snowpark_connect/includes/python/pyspark/streaming/tests/test_listener.py +0 -160
- snowflake/snowpark_connect/includes/python/pyspark/tests/__init__.py +0 -16
- snowflake/snowpark_connect/includes/python/pyspark/tests/test_appsubmit.py +0 -306
- snowflake/snowpark_connect/includes/python/pyspark/tests/test_broadcast.py +0 -196
- snowflake/snowpark_connect/includes/python/pyspark/tests/test_conf.py +0 -44
- snowflake/snowpark_connect/includes/python/pyspark/tests/test_context.py +0 -346
- snowflake/snowpark_connect/includes/python/pyspark/tests/test_daemon.py +0 -89
- snowflake/snowpark_connect/includes/python/pyspark/tests/test_install_spark.py +0 -124
- snowflake/snowpark_connect/includes/python/pyspark/tests/test_join.py +0 -69
- snowflake/snowpark_connect/includes/python/pyspark/tests/test_memory_profiler.py +0 -167
- snowflake/snowpark_connect/includes/python/pyspark/tests/test_pin_thread.py +0 -194
- snowflake/snowpark_connect/includes/python/pyspark/tests/test_profiler.py +0 -168
- snowflake/snowpark_connect/includes/python/pyspark/tests/test_rdd.py +0 -939
- snowflake/snowpark_connect/includes/python/pyspark/tests/test_rddbarrier.py +0 -52
- snowflake/snowpark_connect/includes/python/pyspark/tests/test_rddsampler.py +0 -66
- snowflake/snowpark_connect/includes/python/pyspark/tests/test_readwrite.py +0 -368
- snowflake/snowpark_connect/includes/python/pyspark/tests/test_serializers.py +0 -257
- snowflake/snowpark_connect/includes/python/pyspark/tests/test_shuffle.py +0 -267
- snowflake/snowpark_connect/includes/python/pyspark/tests/test_stage_sched.py +0 -153
- snowflake/snowpark_connect/includes/python/pyspark/tests/test_statcounter.py +0 -130
- snowflake/snowpark_connect/includes/python/pyspark/tests/test_taskcontext.py +0 -350
- snowflake/snowpark_connect/includes/python/pyspark/tests/test_util.py +0 -97
- snowflake/snowpark_connect/includes/python/pyspark/tests/test_worker.py +0 -271
- snowpark_connect-0.24.0.dist-info/RECORD +0 -898
- {snowpark_connect-0.24.0.data → snowpark_connect-0.25.0.data}/scripts/snowpark-connect +0 -0
- {snowpark_connect-0.24.0.data → snowpark_connect-0.25.0.data}/scripts/snowpark-session +0 -0
- {snowpark_connect-0.24.0.data → snowpark_connect-0.25.0.data}/scripts/snowpark-submit +0 -0
- {snowpark_connect-0.24.0.dist-info → snowpark_connect-0.25.0.dist-info}/WHEEL +0 -0
- {snowpark_connect-0.24.0.dist-info → snowpark_connect-0.25.0.dist-info}/licenses/LICENSE-binary +0 -0
- {snowpark_connect-0.24.0.dist-info → snowpark_connect-0.25.0.dist-info}/licenses/LICENSE.txt +0 -0
- {snowpark_connect-0.24.0.dist-info → snowpark_connect-0.25.0.dist-info}/licenses/NOTICE-binary +0 -0
- {snowpark_connect-0.24.0.dist-info → snowpark_connect-0.25.0.dist-info}/top_level.txt +0 -0
|
@@ -1,160 +0,0 @@
|
|
|
1
|
-
#
|
|
2
|
-
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
3
|
-
# contributor license agreements. See the NOTICE file distributed with
|
|
4
|
-
# this work for additional information regarding copyright ownership.
|
|
5
|
-
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
6
|
-
# (the "License"); you may not use this file except in compliance with
|
|
7
|
-
# the License. You may obtain a copy of the License at
|
|
8
|
-
#
|
|
9
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
-
#
|
|
11
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
12
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
-
# See the License for the specific language governing permissions and
|
|
15
|
-
# limitations under the License.
|
|
16
|
-
#
|
|
17
|
-
import unittest
|
|
18
|
-
|
|
19
|
-
import numpy as np
|
|
20
|
-
import pandas as pd
|
|
21
|
-
|
|
22
|
-
from pyspark import pandas as ps
|
|
23
|
-
from pyspark.testing.pandasutils import ComparisonTestBase
|
|
24
|
-
from pyspark.testing.sqlutils import SQLTestUtils
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
class FrameResetIndexMixin:
|
|
28
|
-
@property
|
|
29
|
-
def pdf(self):
|
|
30
|
-
return pd.DataFrame(
|
|
31
|
-
{"a": [1, 2, 3, 4, 5, 6, 7, 8, 9], "b": [4, 5, 6, 3, 2, 1, 0, 0, 0]},
|
|
32
|
-
index=np.random.rand(9),
|
|
33
|
-
)
|
|
34
|
-
|
|
35
|
-
@property
|
|
36
|
-
def df_pair(self):
|
|
37
|
-
pdf = self.pdf
|
|
38
|
-
psdf = ps.from_pandas(pdf)
|
|
39
|
-
return pdf, psdf
|
|
40
|
-
|
|
41
|
-
def test_reset_index(self):
|
|
42
|
-
pdf = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}, index=np.random.rand(3))
|
|
43
|
-
psdf = ps.from_pandas(pdf)
|
|
44
|
-
|
|
45
|
-
self.assert_eq(psdf.reset_index(), pdf.reset_index())
|
|
46
|
-
self.assert_eq(psdf.reset_index().index, pdf.reset_index().index)
|
|
47
|
-
self.assert_eq(psdf.reset_index(drop=True), pdf.reset_index(drop=True))
|
|
48
|
-
|
|
49
|
-
pdf.index.name = "a"
|
|
50
|
-
psdf.index.name = "a"
|
|
51
|
-
|
|
52
|
-
with self.assertRaisesRegex(ValueError, "cannot insert a, already exists"):
|
|
53
|
-
psdf.reset_index()
|
|
54
|
-
|
|
55
|
-
self.assert_eq(psdf.reset_index(drop=True), pdf.reset_index(drop=True))
|
|
56
|
-
|
|
57
|
-
# inplace
|
|
58
|
-
pser = pdf.a
|
|
59
|
-
psser = psdf.a
|
|
60
|
-
pdf.reset_index(drop=True, inplace=True)
|
|
61
|
-
psdf.reset_index(drop=True, inplace=True)
|
|
62
|
-
self.assert_eq(psdf, pdf)
|
|
63
|
-
self.assert_eq(psser, pser)
|
|
64
|
-
|
|
65
|
-
pdf.columns = ["index", "b"]
|
|
66
|
-
psdf.columns = ["index", "b"]
|
|
67
|
-
self.assert_eq(psdf.reset_index(), pdf.reset_index())
|
|
68
|
-
|
|
69
|
-
def test_reset_index_with_default_index_types(self):
|
|
70
|
-
pdf = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}, index=np.random.rand(3))
|
|
71
|
-
psdf = ps.from_pandas(pdf)
|
|
72
|
-
|
|
73
|
-
with ps.option_context("compute.default_index_type", "sequence"):
|
|
74
|
-
self.assert_eq(psdf.reset_index(), pdf.reset_index())
|
|
75
|
-
|
|
76
|
-
with ps.option_context("compute.default_index_type", "distributed-sequence"):
|
|
77
|
-
self.assert_eq(psdf.reset_index(), pdf.reset_index())
|
|
78
|
-
|
|
79
|
-
with ps.option_context("compute.default_index_type", "distributed"):
|
|
80
|
-
# the index is different.
|
|
81
|
-
self.assert_eq(
|
|
82
|
-
psdf.reset_index()._to_pandas().reset_index(drop=True), pdf.reset_index()
|
|
83
|
-
)
|
|
84
|
-
|
|
85
|
-
def test_reset_index_with_multiindex_columns(self):
|
|
86
|
-
index = pd.MultiIndex.from_tuples(
|
|
87
|
-
[("bird", "falcon"), ("bird", "parrot"), ("mammal", "lion"), ("mammal", "monkey")],
|
|
88
|
-
names=["class", "name"],
|
|
89
|
-
)
|
|
90
|
-
columns = pd.MultiIndex.from_tuples([("speed", "max"), ("species", "type")])
|
|
91
|
-
pdf = pd.DataFrame(
|
|
92
|
-
[(389.0, "fly"), (24.0, "fly"), (80.5, "run"), (np.nan, "jump")],
|
|
93
|
-
index=index,
|
|
94
|
-
columns=columns,
|
|
95
|
-
)
|
|
96
|
-
psdf = ps.from_pandas(pdf)
|
|
97
|
-
|
|
98
|
-
self.assert_eq(psdf, pdf)
|
|
99
|
-
self.assert_eq(psdf.reset_index(), pdf.reset_index())
|
|
100
|
-
self.assert_eq(psdf.reset_index(level="class"), pdf.reset_index(level="class"))
|
|
101
|
-
self.assert_eq(
|
|
102
|
-
psdf.reset_index(level="class", col_level=1),
|
|
103
|
-
pdf.reset_index(level="class", col_level=1),
|
|
104
|
-
)
|
|
105
|
-
self.assert_eq(
|
|
106
|
-
psdf.reset_index(level="class", col_level=1, col_fill="species"),
|
|
107
|
-
pdf.reset_index(level="class", col_level=1, col_fill="species"),
|
|
108
|
-
)
|
|
109
|
-
self.assert_eq(
|
|
110
|
-
psdf.reset_index(level="class", col_level=1, col_fill="genus"),
|
|
111
|
-
pdf.reset_index(level="class", col_level=1, col_fill="genus"),
|
|
112
|
-
)
|
|
113
|
-
|
|
114
|
-
with self.assertRaisesRegex(IndexError, "Index has only 2 levels, not 3"):
|
|
115
|
-
psdf.reset_index(col_level=2)
|
|
116
|
-
|
|
117
|
-
pdf.index.names = [("x", "class"), ("y", "name")]
|
|
118
|
-
psdf.index.names = [("x", "class"), ("y", "name")]
|
|
119
|
-
|
|
120
|
-
self.assert_eq(psdf.reset_index(), pdf.reset_index())
|
|
121
|
-
|
|
122
|
-
with self.assertRaisesRegex(ValueError, "Item must have length equal to number of levels."):
|
|
123
|
-
psdf.reset_index(col_level=1)
|
|
124
|
-
|
|
125
|
-
def test_index_to_frame_reset_index(self):
|
|
126
|
-
def check(psdf, pdf):
|
|
127
|
-
self.assert_eq(psdf.reset_index(), pdf.reset_index())
|
|
128
|
-
self.assert_eq(psdf.reset_index(drop=True), pdf.reset_index(drop=True))
|
|
129
|
-
|
|
130
|
-
pdf.reset_index(drop=True, inplace=True)
|
|
131
|
-
psdf.reset_index(drop=True, inplace=True)
|
|
132
|
-
self.assert_eq(psdf, pdf)
|
|
133
|
-
|
|
134
|
-
pdf, psdf = self.df_pair
|
|
135
|
-
check(psdf.index.to_frame(), pdf.index.to_frame())
|
|
136
|
-
check(psdf.index.to_frame(index=False), pdf.index.to_frame(index=False))
|
|
137
|
-
|
|
138
|
-
check(psdf.index.to_frame(name="a"), pdf.index.to_frame(name="a"))
|
|
139
|
-
check(psdf.index.to_frame(index=False, name="a"), pdf.index.to_frame(index=False, name="a"))
|
|
140
|
-
check(psdf.index.to_frame(name=("x", "a")), pdf.index.to_frame(name=("x", "a")))
|
|
141
|
-
check(
|
|
142
|
-
psdf.index.to_frame(index=False, name=("x", "a")),
|
|
143
|
-
pdf.index.to_frame(index=False, name=("x", "a")),
|
|
144
|
-
)
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
class FrameResetIndexTests(FrameResetIndexMixin, ComparisonTestBase, SQLTestUtils):
|
|
148
|
-
pass
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
if __name__ == "__main__":
|
|
152
|
-
from pyspark.pandas.tests.indexes.test_reset_index import * # noqa: F401
|
|
153
|
-
|
|
154
|
-
try:
|
|
155
|
-
import xmlrunner
|
|
156
|
-
|
|
157
|
-
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
|
|
158
|
-
except ImportError:
|
|
159
|
-
testRunner = None
|
|
160
|
-
unittest.main(testRunner=testRunner, verbosity=2)
|
|
@@ -1,128 +0,0 @@
|
|
|
1
|
-
#
|
|
2
|
-
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
3
|
-
# contributor license agreements. See the NOTICE file distributed with
|
|
4
|
-
# this work for additional information regarding copyright ownership.
|
|
5
|
-
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
6
|
-
# (the "License"); you may not use this file except in compliance with
|
|
7
|
-
# the License. You may obtain a copy of the License at
|
|
8
|
-
#
|
|
9
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
-
#
|
|
11
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
12
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
-
# See the License for the specific language governing permissions and
|
|
15
|
-
# limitations under the License.
|
|
16
|
-
#
|
|
17
|
-
|
|
18
|
-
import unittest
|
|
19
|
-
from datetime import timedelta
|
|
20
|
-
from distutils.version import LooseVersion
|
|
21
|
-
|
|
22
|
-
import pandas as pd
|
|
23
|
-
|
|
24
|
-
import pyspark.pandas as ps
|
|
25
|
-
from pyspark.testing.pandasutils import PandasOnSparkTestCase, TestUtils
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
class TimedeltaIndexTestsMixin:
|
|
29
|
-
@property
|
|
30
|
-
def pidx(self):
|
|
31
|
-
return pd.TimedeltaIndex(
|
|
32
|
-
[
|
|
33
|
-
timedelta(days=1),
|
|
34
|
-
timedelta(seconds=1),
|
|
35
|
-
timedelta(microseconds=1),
|
|
36
|
-
timedelta(milliseconds=1),
|
|
37
|
-
timedelta(minutes=1),
|
|
38
|
-
timedelta(hours=1),
|
|
39
|
-
timedelta(weeks=1),
|
|
40
|
-
],
|
|
41
|
-
name="x",
|
|
42
|
-
)
|
|
43
|
-
|
|
44
|
-
@property
|
|
45
|
-
def neg_pidx(self):
|
|
46
|
-
return pd.TimedeltaIndex(
|
|
47
|
-
[
|
|
48
|
-
timedelta(days=-1),
|
|
49
|
-
timedelta(seconds=-1),
|
|
50
|
-
timedelta(microseconds=-1),
|
|
51
|
-
timedelta(milliseconds=-1),
|
|
52
|
-
timedelta(minutes=-1),
|
|
53
|
-
timedelta(hours=-1),
|
|
54
|
-
timedelta(weeks=-1),
|
|
55
|
-
],
|
|
56
|
-
name="x",
|
|
57
|
-
)
|
|
58
|
-
|
|
59
|
-
@property
|
|
60
|
-
def psidx(self):
|
|
61
|
-
return ps.from_pandas(self.pidx)
|
|
62
|
-
|
|
63
|
-
@property
|
|
64
|
-
def neg_psidx(self):
|
|
65
|
-
return ps.from_pandas(self.neg_pidx)
|
|
66
|
-
|
|
67
|
-
def test_timedelta_index(self):
|
|
68
|
-
# Create TimedeltaIndex from constructor
|
|
69
|
-
psidx = ps.TimedeltaIndex(
|
|
70
|
-
[
|
|
71
|
-
timedelta(days=1),
|
|
72
|
-
timedelta(seconds=1),
|
|
73
|
-
timedelta(microseconds=1),
|
|
74
|
-
timedelta(milliseconds=1),
|
|
75
|
-
timedelta(minutes=1),
|
|
76
|
-
timedelta(hours=1),
|
|
77
|
-
timedelta(weeks=1),
|
|
78
|
-
],
|
|
79
|
-
name="x",
|
|
80
|
-
)
|
|
81
|
-
self.assert_eq(psidx, self.pidx)
|
|
82
|
-
# Create TimedeltaIndex from Series
|
|
83
|
-
self.assert_eq(
|
|
84
|
-
ps.TimedeltaIndex(ps.Series([timedelta(days=1)])),
|
|
85
|
-
pd.TimedeltaIndex(pd.Series([timedelta(days=1)])),
|
|
86
|
-
)
|
|
87
|
-
# Create TimedeltaIndex from Index
|
|
88
|
-
self.assert_eq(
|
|
89
|
-
ps.TimedeltaIndex(ps.Index([timedelta(days=1)])),
|
|
90
|
-
pd.TimedeltaIndex(pd.Index([timedelta(days=1)])),
|
|
91
|
-
)
|
|
92
|
-
|
|
93
|
-
# ps.TimedeltaIndex(ps.Index([1, 2, 3]))
|
|
94
|
-
with self.assertRaisesRegexp(TypeError, "Index.name must be a hashable type"):
|
|
95
|
-
ps.TimedeltaIndex([timedelta(1), timedelta(microseconds=2)], name=[(1, 2)])
|
|
96
|
-
with self.assertRaisesRegexp(
|
|
97
|
-
TypeError, "Cannot perform 'all' with this index type: TimedeltaIndex"
|
|
98
|
-
):
|
|
99
|
-
psidx.all()
|
|
100
|
-
|
|
101
|
-
@unittest.skipIf(
|
|
102
|
-
LooseVersion(pd.__version__) >= LooseVersion("2.0.0"),
|
|
103
|
-
"TODO(SPARK-43705): Enable TimedeltaIndexTests.test_properties for pandas 2.0.0.",
|
|
104
|
-
)
|
|
105
|
-
def test_properties(self):
|
|
106
|
-
self.assert_eq(self.psidx.days, self.pidx.days)
|
|
107
|
-
self.assert_eq(self.psidx.seconds, self.pidx.seconds)
|
|
108
|
-
self.assert_eq(self.psidx.microseconds, self.pidx.microseconds)
|
|
109
|
-
self.assert_eq(self.neg_psidx.days, self.neg_pidx.days)
|
|
110
|
-
self.assert_eq(self.neg_psidx.seconds, self.neg_pidx.seconds)
|
|
111
|
-
self.assert_eq(self.neg_psidx.microseconds, self.neg_pidx.microseconds)
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
class TimedeltaIndexTests(TimedeltaIndexTestsMixin, PandasOnSparkTestCase, TestUtils):
|
|
115
|
-
pass
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
if __name__ == "__main__":
|
|
119
|
-
import unittest
|
|
120
|
-
from pyspark.pandas.tests.indexes.test_timedelta import * # noqa: F401
|
|
121
|
-
|
|
122
|
-
try:
|
|
123
|
-
import xmlrunner
|
|
124
|
-
|
|
125
|
-
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
|
|
126
|
-
except ImportError:
|
|
127
|
-
testRunner = None
|
|
128
|
-
unittest.main(testRunner=testRunner, verbosity=2)
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
#
|
|
2
|
-
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
3
|
-
# contributor license agreements. See the NOTICE file distributed with
|
|
4
|
-
# this work for additional information regarding copyright ownership.
|
|
5
|
-
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
6
|
-
# (the "License"); you may not use this file except in compliance with
|
|
7
|
-
# the License. You may obtain a copy of the License at
|
|
8
|
-
#
|
|
9
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
-
#
|
|
11
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
12
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
-
# See the License for the specific language governing permissions and
|
|
15
|
-
# limitations under the License.
|
|
16
|
-
#
|
|
@@ -1,137 +0,0 @@
|
|
|
1
|
-
#
|
|
2
|
-
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
3
|
-
# contributor license agreements. See the NOTICE file distributed with
|
|
4
|
-
# this work for additional information regarding copyright ownership.
|
|
5
|
-
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
6
|
-
# (the "License"); you may not use this file except in compliance with
|
|
7
|
-
# the License. You may obtain a copy of the License at
|
|
8
|
-
#
|
|
9
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
-
#
|
|
11
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
12
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
-
# See the License for the specific language governing permissions and
|
|
15
|
-
# limitations under the License.
|
|
16
|
-
#
|
|
17
|
-
from distutils.version import LooseVersion
|
|
18
|
-
import unittest
|
|
19
|
-
|
|
20
|
-
import numpy as np
|
|
21
|
-
import pandas as pd
|
|
22
|
-
|
|
23
|
-
from pyspark import pandas as ps
|
|
24
|
-
from pyspark.testing.pandasutils import (
|
|
25
|
-
have_tabulate,
|
|
26
|
-
ComparisonTestBase,
|
|
27
|
-
tabulate_requirement_message,
|
|
28
|
-
)
|
|
29
|
-
from pyspark.testing.sqlutils import SQLTestUtils
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
# This file contains test cases for 'Serialization / IO / Conversion'
|
|
33
|
-
# https://spark.apache.org/docs/latest/api/python/reference/pyspark.pandas/frame.html#serialization-io-conversion
|
|
34
|
-
class FrameIOMixin:
|
|
35
|
-
@property
|
|
36
|
-
def pdf(self):
|
|
37
|
-
return pd.DataFrame(
|
|
38
|
-
{"a": [1, 2, 3, 4, 5, 6, 7, 8, 9], "b": [4, 5, 6, 3, 2, 1, 0, 0, 0]},
|
|
39
|
-
index=np.random.rand(9),
|
|
40
|
-
)
|
|
41
|
-
|
|
42
|
-
@property
|
|
43
|
-
def df_pair(self):
|
|
44
|
-
pdf = self.pdf
|
|
45
|
-
psdf = ps.from_pandas(pdf)
|
|
46
|
-
return pdf, psdf
|
|
47
|
-
|
|
48
|
-
def test_to_numpy(self):
|
|
49
|
-
pdf = pd.DataFrame(
|
|
50
|
-
{
|
|
51
|
-
"a": [4, 2, 3, 4, 8, 6],
|
|
52
|
-
"b": [1, 2, 9, 4, 2, 4],
|
|
53
|
-
"c": ["one", "three", "six", "seven", "one", "5"],
|
|
54
|
-
},
|
|
55
|
-
index=np.random.rand(6),
|
|
56
|
-
)
|
|
57
|
-
|
|
58
|
-
psdf = ps.from_pandas(pdf)
|
|
59
|
-
|
|
60
|
-
self.assert_eq(psdf.to_numpy(), pdf.values)
|
|
61
|
-
|
|
62
|
-
def test_to_pandas(self):
|
|
63
|
-
pdf, psdf = self.df_pair
|
|
64
|
-
self.assert_eq(psdf._to_pandas(), pdf)
|
|
65
|
-
|
|
66
|
-
def test_to_spark(self):
|
|
67
|
-
psdf = ps.from_pandas(self.pdf)
|
|
68
|
-
|
|
69
|
-
with self.assertRaisesRegex(ValueError, "'index_col' cannot be overlapped"):
|
|
70
|
-
psdf.to_spark(index_col="a")
|
|
71
|
-
|
|
72
|
-
with self.assertRaisesRegex(ValueError, "length of index columns.*1.*3"):
|
|
73
|
-
psdf.to_spark(index_col=["x", "y", "z"])
|
|
74
|
-
|
|
75
|
-
@unittest.skipIf(not have_tabulate, tabulate_requirement_message)
|
|
76
|
-
def test_to_markdown(self):
|
|
77
|
-
pdf = pd.DataFrame(data={"animal_1": ["elk", "pig"], "animal_2": ["dog", "quetzal"]})
|
|
78
|
-
psdf = ps.from_pandas(pdf)
|
|
79
|
-
|
|
80
|
-
self.assert_eq(pdf.to_markdown(), psdf.to_markdown())
|
|
81
|
-
|
|
82
|
-
def test_from_dict(self):
|
|
83
|
-
data = {"row_1": [3, 2, 1, 0], "row_2": [10, 20, 30, 40]}
|
|
84
|
-
pdf = pd.DataFrame.from_dict(data)
|
|
85
|
-
psdf = ps.DataFrame.from_dict(data)
|
|
86
|
-
self.assert_eq(pdf, psdf)
|
|
87
|
-
|
|
88
|
-
pdf = pd.DataFrame.from_dict(data, dtype="int8")
|
|
89
|
-
psdf = ps.DataFrame.from_dict(data, dtype="int8")
|
|
90
|
-
self.assert_eq(pdf, psdf)
|
|
91
|
-
|
|
92
|
-
pdf = pd.DataFrame.from_dict(data, orient="index", columns=["A", "B", "C", "D"])
|
|
93
|
-
psdf = ps.DataFrame.from_dict(data, orient="index", columns=["A", "B", "C", "D"])
|
|
94
|
-
self.assert_eq(pdf, psdf)
|
|
95
|
-
|
|
96
|
-
@unittest.skipIf(
|
|
97
|
-
LooseVersion(pd.__version__) < LooseVersion("1.3.0"),
|
|
98
|
-
"pandas support `Styler.to_latex` since 1.3.0",
|
|
99
|
-
)
|
|
100
|
-
def test_style(self):
|
|
101
|
-
# Currently, the `style` function returns a pandas object `Styler` as it is,
|
|
102
|
-
# processing only the number of rows declared in `compute.max_rows`.
|
|
103
|
-
# So it's a bit vague to test, but we are doing minimal tests instead of not testing at all.
|
|
104
|
-
pdf = pd.DataFrame(np.random.randn(10, 4), columns=["A", "B", "C", "D"])
|
|
105
|
-
psdf = ps.from_pandas(pdf)
|
|
106
|
-
|
|
107
|
-
def style_negative(v, props=""):
|
|
108
|
-
return props if v < 0 else None
|
|
109
|
-
|
|
110
|
-
def check_style():
|
|
111
|
-
# If the value is negative, the text color will be displayed as red.
|
|
112
|
-
pdf_style = pdf.style.applymap(style_negative, props="color:red;")
|
|
113
|
-
psdf_style = psdf.style.applymap(style_negative, props="color:red;")
|
|
114
|
-
|
|
115
|
-
# Test whether the same shape as pandas table is created including the color.
|
|
116
|
-
self.assert_eq(pdf_style.to_latex(), psdf_style.to_latex())
|
|
117
|
-
|
|
118
|
-
check_style()
|
|
119
|
-
|
|
120
|
-
with ps.option_context("compute.max_rows", None):
|
|
121
|
-
check_style()
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
class FrameIOTests(FrameIOMixin, ComparisonTestBase, SQLTestUtils):
|
|
125
|
-
pass
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
if __name__ == "__main__":
|
|
129
|
-
from pyspark.pandas.tests.io.test_io import * # noqa: F401
|
|
130
|
-
|
|
131
|
-
try:
|
|
132
|
-
import xmlrunner
|
|
133
|
-
|
|
134
|
-
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
|
|
135
|
-
except ImportError:
|
|
136
|
-
testRunner = None
|
|
137
|
-
unittest.main(testRunner=testRunner, verbosity=2)
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
#
|
|
2
|
-
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
3
|
-
# contributor license agreements. See the NOTICE file distributed with
|
|
4
|
-
# this work for additional information regarding copyright ownership.
|
|
5
|
-
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
6
|
-
# (the "License"); you may not use this file except in compliance with
|
|
7
|
-
# the License. You may obtain a copy of the License at
|
|
8
|
-
#
|
|
9
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
-
#
|
|
11
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
12
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
-
# See the License for the specific language governing permissions and
|
|
15
|
-
# limitations under the License.
|
|
16
|
-
#
|
|
@@ -1,170 +0,0 @@
|
|
|
1
|
-
#
|
|
2
|
-
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
3
|
-
# contributor license agreements. See the NOTICE file distributed with
|
|
4
|
-
# this work for additional information regarding copyright ownership.
|
|
5
|
-
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
6
|
-
# (the "License"); you may not use this file except in compliance with
|
|
7
|
-
# the License. You may obtain a copy of the License at
|
|
8
|
-
#
|
|
9
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
-
#
|
|
11
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
12
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
-
# See the License for the specific language governing permissions and
|
|
15
|
-
# limitations under the License.
|
|
16
|
-
#
|
|
17
|
-
|
|
18
|
-
import pandas as pd
|
|
19
|
-
import numpy as np
|
|
20
|
-
|
|
21
|
-
from pyspark import pandas as ps
|
|
22
|
-
from pyspark.pandas.config import set_option, reset_option, option_context
|
|
23
|
-
from pyspark.pandas.plot import TopNPlotBase, SampledPlotBase, HistogramPlotBase, BoxPlotBase
|
|
24
|
-
from pyspark.pandas.exceptions import PandasNotImplementedError
|
|
25
|
-
from pyspark.testing.pandasutils import PandasOnSparkTestCase
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
class DataFramePlotTestsMixin:
|
|
29
|
-
@classmethod
|
|
30
|
-
def setUpClass(cls):
|
|
31
|
-
super().setUpClass()
|
|
32
|
-
set_option("plotting.max_rows", 2000)
|
|
33
|
-
set_option("plotting.sample_ratio", None)
|
|
34
|
-
|
|
35
|
-
@classmethod
|
|
36
|
-
def tearDownClass(cls):
|
|
37
|
-
reset_option("plotting.max_rows")
|
|
38
|
-
reset_option("plotting.sample_ratio")
|
|
39
|
-
super().tearDownClass()
|
|
40
|
-
|
|
41
|
-
def test_missing(self):
|
|
42
|
-
psdf = ps.DataFrame(np.random.rand(2500, 4), columns=["a", "b", "c", "d"])
|
|
43
|
-
|
|
44
|
-
unsupported_functions = ["hexbin"]
|
|
45
|
-
|
|
46
|
-
for name in unsupported_functions:
|
|
47
|
-
with self.assertRaisesRegex(
|
|
48
|
-
PandasNotImplementedError, "method.*DataFrame.*{}.*not implemented".format(name)
|
|
49
|
-
):
|
|
50
|
-
getattr(psdf.plot, name)()
|
|
51
|
-
|
|
52
|
-
def test_topn_max_rows(self):
|
|
53
|
-
|
|
54
|
-
pdf = pd.DataFrame(np.random.rand(2500, 4), columns=["a", "b", "c", "d"])
|
|
55
|
-
psdf = ps.from_pandas(pdf)
|
|
56
|
-
|
|
57
|
-
data = TopNPlotBase().get_top_n(psdf)
|
|
58
|
-
self.assertEqual(len(data), 2000)
|
|
59
|
-
|
|
60
|
-
def test_sampled_plot_with_ratio(self):
|
|
61
|
-
with option_context("plotting.sample_ratio", 0.5):
|
|
62
|
-
pdf = pd.DataFrame(np.random.rand(2500, 4), columns=["a", "b", "c", "d"])
|
|
63
|
-
psdf = ps.from_pandas(pdf)
|
|
64
|
-
data = SampledPlotBase().get_sampled(psdf)
|
|
65
|
-
self.assertEqual(round(len(data) / 2500, 1), 0.5)
|
|
66
|
-
|
|
67
|
-
def test_sampled_plot_with_max_rows(self):
|
|
68
|
-
# 'plotting.max_rows' is 2000
|
|
69
|
-
pdf = pd.DataFrame(np.random.rand(2000, 4), columns=["a", "b", "c", "d"])
|
|
70
|
-
psdf = ps.from_pandas(pdf)
|
|
71
|
-
data = SampledPlotBase().get_sampled(psdf)
|
|
72
|
-
self.assertEqual(round(len(data) / 2000, 1), 1)
|
|
73
|
-
|
|
74
|
-
def test_compute_hist_single_column(self):
|
|
75
|
-
psdf = ps.DataFrame(
|
|
76
|
-
{"a": [1, 2, 3, 4, 5, 6, 7, 8, 9, 15, 50]}, index=[0, 1, 3, 5, 6, 8, 9, 9, 9, 10, 10]
|
|
77
|
-
)
|
|
78
|
-
|
|
79
|
-
expected_bins = np.linspace(1, 50, 11)
|
|
80
|
-
bins = HistogramPlotBase.get_bins(psdf[["a"]].to_spark(), 10)
|
|
81
|
-
|
|
82
|
-
expected_histogram = np.array([5, 4, 1, 0, 0, 0, 0, 0, 0, 1])
|
|
83
|
-
histogram = HistogramPlotBase.compute_hist(psdf[["a"]], bins)[0]
|
|
84
|
-
self.assert_eq(pd.Series(expected_bins), pd.Series(bins))
|
|
85
|
-
self.assert_eq(pd.Series(expected_histogram, name="a"), histogram, almost=True)
|
|
86
|
-
|
|
87
|
-
def test_compute_hist_multi_columns(self):
|
|
88
|
-
expected_bins = np.linspace(1, 50, 11)
|
|
89
|
-
psdf = ps.DataFrame(
|
|
90
|
-
{
|
|
91
|
-
"a": [1, 2, 3, 4, 5, 6, 7, 8, 9, 15, 50],
|
|
92
|
-
"b": [50, 50, 30, 30, 30, 24, 10, 5, 4, 3, 1],
|
|
93
|
-
}
|
|
94
|
-
)
|
|
95
|
-
|
|
96
|
-
bins = HistogramPlotBase.get_bins(psdf.to_spark(), 10)
|
|
97
|
-
self.assert_eq(pd.Series(expected_bins), pd.Series(bins))
|
|
98
|
-
|
|
99
|
-
expected_histograms = [
|
|
100
|
-
np.array([5, 4, 1, 0, 0, 0, 0, 0, 0, 1]),
|
|
101
|
-
np.array([4, 1, 0, 0, 1, 3, 0, 0, 0, 2]),
|
|
102
|
-
]
|
|
103
|
-
histograms = HistogramPlotBase.compute_hist(psdf, bins)
|
|
104
|
-
expected_names = ["a", "b"]
|
|
105
|
-
|
|
106
|
-
for histogram, expected_histogram, expected_name in zip(
|
|
107
|
-
histograms, expected_histograms, expected_names
|
|
108
|
-
):
|
|
109
|
-
self.assert_eq(
|
|
110
|
-
pd.Series(expected_histogram, name=expected_name), histogram, almost=True
|
|
111
|
-
)
|
|
112
|
-
|
|
113
|
-
def test_compute_box_multi_columns(self):
|
|
114
|
-
# compare compute_multicol_stats with compute_stats
|
|
115
|
-
def check_box_multi_columns(psdf):
|
|
116
|
-
k = 1.5
|
|
117
|
-
multicol_stats = BoxPlotBase.compute_multicol_stats(
|
|
118
|
-
psdf, ["a", "b", "c"], whis=k, precision=0.01
|
|
119
|
-
)
|
|
120
|
-
multicol_outliers = BoxPlotBase.multicol_outliers(psdf, multicol_stats)
|
|
121
|
-
multicol_whiskers = BoxPlotBase.calc_multicol_whiskers(
|
|
122
|
-
["a", "b", "c"], multicol_outliers
|
|
123
|
-
)
|
|
124
|
-
|
|
125
|
-
for col in ["a", "b", "c"]:
|
|
126
|
-
col_stats = multicol_stats[col]
|
|
127
|
-
col_whiskers = multicol_whiskers[col]
|
|
128
|
-
|
|
129
|
-
stats, fences = BoxPlotBase.compute_stats(psdf[col], col, whis=k, precision=0.01)
|
|
130
|
-
outliers = BoxPlotBase.outliers(psdf[col], col, *fences)
|
|
131
|
-
whiskers = BoxPlotBase.calc_whiskers(col, outliers)
|
|
132
|
-
|
|
133
|
-
self.assertEqual(stats["mean"], col_stats["mean"])
|
|
134
|
-
self.assertEqual(stats["med"], col_stats["med"])
|
|
135
|
-
self.assertEqual(stats["q1"], col_stats["q1"])
|
|
136
|
-
self.assertEqual(stats["q3"], col_stats["q3"])
|
|
137
|
-
self.assertEqual(fences[0], col_stats["lfence"])
|
|
138
|
-
self.assertEqual(fences[1], col_stats["ufence"])
|
|
139
|
-
self.assertEqual(whiskers[0], col_whiskers["min"])
|
|
140
|
-
self.assertEqual(whiskers[1], col_whiskers["max"])
|
|
141
|
-
|
|
142
|
-
pdf = pd.DataFrame(
|
|
143
|
-
{
|
|
144
|
-
"a": [1, 2, 3, 4, 5, 6, 7, 8, 9, 15, 50],
|
|
145
|
-
"b": [3, 2, 5, 4, 5, 6, 8, 8, 11, 60, 90],
|
|
146
|
-
"c": [-30, -2, 5, 4, 5, 6, -8, 8, 11, 12, 18],
|
|
147
|
-
},
|
|
148
|
-
index=[0, 1, 3, 5, 6, 8, 9, 9, 9, 10, 10],
|
|
149
|
-
)
|
|
150
|
-
psdf = ps.from_pandas(pdf)
|
|
151
|
-
|
|
152
|
-
check_box_multi_columns(psdf)
|
|
153
|
-
check_box_multi_columns(-psdf)
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
class DataFramePlotTests(DataFramePlotTestsMixin, PandasOnSparkTestCase):
|
|
157
|
-
pass
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
if __name__ == "__main__":
|
|
161
|
-
import unittest
|
|
162
|
-
from pyspark.pandas.tests.plot.test_frame_plot import * # noqa: F401
|
|
163
|
-
|
|
164
|
-
try:
|
|
165
|
-
import xmlrunner
|
|
166
|
-
|
|
167
|
-
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
|
|
168
|
-
except ImportError:
|
|
169
|
-
testRunner = None
|
|
170
|
-
unittest.main(testRunner=testRunner, verbosity=2)
|