snowpark-connect 0.24.0__py3-none-any.whl → 0.25.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of snowpark-connect might be problematic. Click here for more details.

Files changed (474) hide show
  1. snowflake/snowpark_connect/column_name_handler.py +116 -4
  2. snowflake/snowpark_connect/config.py +13 -0
  3. snowflake/snowpark_connect/constants.py +0 -29
  4. snowflake/snowpark_connect/dataframe_container.py +6 -0
  5. snowflake/snowpark_connect/execute_plan/map_execution_command.py +56 -1
  6. snowflake/snowpark_connect/expression/literal.py +13 -2
  7. snowflake/snowpark_connect/expression/map_cast.py +5 -8
  8. snowflake/snowpark_connect/expression/map_sql_expression.py +23 -1
  9. snowflake/snowpark_connect/expression/map_udf.py +26 -8
  10. snowflake/snowpark_connect/expression/map_unresolved_attribute.py +199 -15
  11. snowflake/snowpark_connect/expression/map_unresolved_extract_value.py +44 -16
  12. snowflake/snowpark_connect/expression/map_unresolved_function.py +825 -353
  13. snowflake/snowpark_connect/expression/map_unresolved_star.py +3 -2
  14. snowflake/snowpark_connect/hidden_column.py +39 -0
  15. snowflake/snowpark_connect/includes/jars/hadoop-client-api-trimmed-3.3.4.jar +0 -0
  16. snowflake/snowpark_connect/includes/jars/{hadoop-client-api-3.3.4.jar → spark-connect-client-jvm_2.12-3.5.6.jar} +0 -0
  17. snowflake/snowpark_connect/relation/map_column_ops.py +17 -4
  18. snowflake/snowpark_connect/relation/map_extension.py +52 -11
  19. snowflake/snowpark_connect/relation/map_join.py +258 -62
  20. snowflake/snowpark_connect/relation/map_sql.py +88 -11
  21. snowflake/snowpark_connect/relation/map_udtf.py +4 -2
  22. snowflake/snowpark_connect/relation/read/map_read.py +3 -3
  23. snowflake/snowpark_connect/relation/read/map_read_jdbc.py +1 -1
  24. snowflake/snowpark_connect/relation/read/map_read_json.py +8 -1
  25. snowflake/snowpark_connect/relation/read/map_read_table.py +1 -9
  26. snowflake/snowpark_connect/relation/read/reader_config.py +3 -1
  27. snowflake/snowpark_connect/relation/write/map_write.py +62 -53
  28. snowflake/snowpark_connect/resources_initializer.py +29 -1
  29. snowflake/snowpark_connect/server.py +18 -3
  30. snowflake/snowpark_connect/type_mapping.py +29 -25
  31. snowflake/snowpark_connect/typed_column.py +14 -0
  32. snowflake/snowpark_connect/utils/artifacts.py +23 -0
  33. snowflake/snowpark_connect/utils/context.py +6 -1
  34. snowflake/snowpark_connect/utils/scala_udf_utils.py +588 -0
  35. snowflake/snowpark_connect/utils/telemetry.py +6 -17
  36. snowflake/snowpark_connect/utils/udf_helper.py +2 -0
  37. snowflake/snowpark_connect/utils/udf_utils.py +38 -7
  38. snowflake/snowpark_connect/utils/udtf_utils.py +17 -3
  39. snowflake/snowpark_connect/version.py +1 -1
  40. {snowpark_connect-0.24.0.dist-info → snowpark_connect-0.25.0.dist-info}/METADATA +1 -1
  41. snowpark_connect-0.25.0.dist-info/RECORD +477 -0
  42. snowflake/snowpark_connect/includes/jars/scala-compiler-2.12.18.jar +0 -0
  43. snowflake/snowpark_connect/includes/jars/spark-kubernetes_2.12-3.5.6.jar +0 -0
  44. snowflake/snowpark_connect/includes/jars/spark-mllib_2.12-3.5.6.jar +0 -0
  45. snowflake/snowpark_connect/includes/jars/spark-streaming_2.12-3.5.6.jar +0 -0
  46. snowflake/snowpark_connect/includes/python/pyspark/errors/tests/__init__.py +0 -16
  47. snowflake/snowpark_connect/includes/python/pyspark/errors/tests/test_errors.py +0 -60
  48. snowflake/snowpark_connect/includes/python/pyspark/ml/deepspeed/tests/test_deepspeed_distributor.py +0 -306
  49. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/__init__.py +0 -16
  50. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_connect_classification.py +0 -53
  51. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_connect_evaluation.py +0 -50
  52. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_connect_feature.py +0 -43
  53. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_connect_function.py +0 -114
  54. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_connect_pipeline.py +0 -47
  55. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_connect_summarizer.py +0 -43
  56. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_connect_tuning.py +0 -46
  57. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_legacy_mode_classification.py +0 -238
  58. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_legacy_mode_evaluation.py +0 -194
  59. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_legacy_mode_feature.py +0 -156
  60. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_legacy_mode_pipeline.py +0 -184
  61. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_legacy_mode_summarizer.py +0 -78
  62. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_legacy_mode_tuning.py +0 -292
  63. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_parity_torch_data_loader.py +0 -50
  64. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_parity_torch_distributor.py +0 -152
  65. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_algorithms.py +0 -456
  66. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_base.py +0 -96
  67. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_dl_util.py +0 -186
  68. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_evaluation.py +0 -77
  69. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_feature.py +0 -401
  70. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_functions.py +0 -528
  71. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_image.py +0 -82
  72. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_linalg.py +0 -409
  73. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_model_cache.py +0 -55
  74. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_param.py +0 -441
  75. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_persistence.py +0 -546
  76. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_pipeline.py +0 -71
  77. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_stat.py +0 -52
  78. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_training_summary.py +0 -494
  79. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_util.py +0 -85
  80. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_wrapper.py +0 -138
  81. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/__init__.py +0 -16
  82. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/test_cv_io_basic.py +0 -151
  83. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/test_cv_io_nested.py +0 -97
  84. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/test_cv_io_pipeline.py +0 -143
  85. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/test_tuning.py +0 -551
  86. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/test_tvs_io_basic.py +0 -137
  87. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/test_tvs_io_nested.py +0 -96
  88. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/test_tvs_io_pipeline.py +0 -142
  89. snowflake/snowpark_connect/includes/python/pyspark/ml/torch/tests/__init__.py +0 -16
  90. snowflake/snowpark_connect/includes/python/pyspark/ml/torch/tests/test_data_loader.py +0 -137
  91. snowflake/snowpark_connect/includes/python/pyspark/ml/torch/tests/test_distributor.py +0 -561
  92. snowflake/snowpark_connect/includes/python/pyspark/ml/torch/tests/test_log_communication.py +0 -172
  93. snowflake/snowpark_connect/includes/python/pyspark/mllib/tests/__init__.py +0 -16
  94. snowflake/snowpark_connect/includes/python/pyspark/mllib/tests/test_algorithms.py +0 -353
  95. snowflake/snowpark_connect/includes/python/pyspark/mllib/tests/test_feature.py +0 -192
  96. snowflake/snowpark_connect/includes/python/pyspark/mllib/tests/test_linalg.py +0 -680
  97. snowflake/snowpark_connect/includes/python/pyspark/mllib/tests/test_stat.py +0 -206
  98. snowflake/snowpark_connect/includes/python/pyspark/mllib/tests/test_streaming_algorithms.py +0 -471
  99. snowflake/snowpark_connect/includes/python/pyspark/mllib/tests/test_util.py +0 -108
  100. snowflake/snowpark_connect/includes/python/pyspark/pandas/spark/__init__.py +0 -16
  101. snowflake/snowpark_connect/includes/python/pyspark/pandas/spark/accessors.py +0 -1281
  102. snowflake/snowpark_connect/includes/python/pyspark/pandas/spark/functions.py +0 -203
  103. snowflake/snowpark_connect/includes/python/pyspark/pandas/spark/utils.py +0 -202
  104. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/__init__.py +0 -16
  105. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/__init__.py +0 -16
  106. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_any_all.py +0 -177
  107. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_apply_func.py +0 -575
  108. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_binary_ops.py +0 -235
  109. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_combine.py +0 -653
  110. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_compute.py +0 -463
  111. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_corrwith.py +0 -86
  112. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_cov.py +0 -151
  113. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_cumulative.py +0 -139
  114. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_describe.py +0 -458
  115. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_eval.py +0 -86
  116. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_melt.py +0 -202
  117. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_missing_data.py +0 -520
  118. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_pivot.py +0 -361
  119. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/__init__.py +0 -16
  120. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/__init__.py +0 -16
  121. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_any_all.py +0 -40
  122. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_apply_func.py +0 -42
  123. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_binary_ops.py +0 -40
  124. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_combine.py +0 -37
  125. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_compute.py +0 -60
  126. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_corrwith.py +0 -40
  127. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_cov.py +0 -40
  128. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_cumulative.py +0 -90
  129. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_describe.py +0 -40
  130. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_eval.py +0 -40
  131. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_melt.py +0 -40
  132. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_missing_data.py +0 -42
  133. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_pivot.py +0 -37
  134. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/__init__.py +0 -16
  135. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_base.py +0 -36
  136. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_binary_ops.py +0 -42
  137. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_boolean_ops.py +0 -47
  138. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_categorical_ops.py +0 -55
  139. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_complex_ops.py +0 -40
  140. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_date_ops.py +0 -47
  141. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_datetime_ops.py +0 -47
  142. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_null_ops.py +0 -42
  143. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_num_arithmetic.py +0 -43
  144. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_num_ops.py +0 -47
  145. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_num_reverse.py +0 -43
  146. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_string_ops.py +0 -47
  147. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_timedelta_ops.py +0 -47
  148. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_udt_ops.py +0 -40
  149. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/testing_utils.py +0 -226
  150. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/__init__.py +0 -16
  151. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_align.py +0 -39
  152. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_basic_slow.py +0 -55
  153. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_cov_corrwith.py +0 -39
  154. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_dot_frame.py +0 -39
  155. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_dot_series.py +0 -39
  156. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_index.py +0 -39
  157. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_series.py +0 -39
  158. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_setitem_frame.py +0 -43
  159. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_setitem_series.py +0 -43
  160. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/__init__.py +0 -16
  161. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_attrs.py +0 -40
  162. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_constructor.py +0 -39
  163. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_conversion.py +0 -42
  164. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_reindexing.py +0 -42
  165. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_reshaping.py +0 -37
  166. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_spark.py +0 -40
  167. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_take.py +0 -42
  168. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_time_series.py +0 -48
  169. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_truncate.py +0 -40
  170. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/__init__.py +0 -16
  171. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_aggregate.py +0 -40
  172. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_apply_func.py +0 -41
  173. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_cumulative.py +0 -67
  174. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_describe.py +0 -40
  175. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_groupby.py +0 -55
  176. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_head_tail.py +0 -40
  177. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_index.py +0 -38
  178. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_missing_data.py +0 -55
  179. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_split_apply.py +0 -39
  180. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_stat.py +0 -38
  181. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/__init__.py +0 -16
  182. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_align.py +0 -40
  183. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_base.py +0 -50
  184. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_category.py +0 -73
  185. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_datetime.py +0 -39
  186. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_indexing.py +0 -40
  187. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_reindex.py +0 -40
  188. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_rename.py +0 -40
  189. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_reset_index.py +0 -48
  190. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_timedelta.py +0 -39
  191. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/io/__init__.py +0 -16
  192. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/io/test_parity_io.py +0 -40
  193. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/plot/__init__.py +0 -16
  194. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot.py +0 -45
  195. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_matplotlib.py +0 -45
  196. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_plotly.py +0 -49
  197. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot.py +0 -37
  198. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_matplotlib.py +0 -53
  199. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_plotly.py +0 -45
  200. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/__init__.py +0 -16
  201. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_all_any.py +0 -38
  202. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_arg_ops.py +0 -37
  203. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_as_of.py +0 -37
  204. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_as_type.py +0 -38
  205. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_compute.py +0 -37
  206. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_conversion.py +0 -40
  207. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_cumulative.py +0 -40
  208. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_index.py +0 -38
  209. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_missing_data.py +0 -40
  210. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_series.py +0 -37
  211. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_sort.py +0 -38
  212. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_stat.py +0 -38
  213. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_categorical.py +0 -66
  214. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_config.py +0 -37
  215. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_csv.py +0 -37
  216. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_dataframe_conversion.py +0 -42
  217. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_dataframe_spark_io.py +0 -39
  218. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_default_index.py +0 -49
  219. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_ewm.py +0 -37
  220. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_expanding.py +0 -39
  221. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_extension.py +0 -49
  222. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_frame_spark.py +0 -53
  223. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_generic_functions.py +0 -43
  224. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_indexing.py +0 -49
  225. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_indexops_spark.py +0 -39
  226. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_internal.py +0 -41
  227. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_namespace.py +0 -39
  228. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_numpy_compat.py +0 -60
  229. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_ops_on_diff_frames.py +0 -48
  230. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_ops_on_diff_frames_groupby.py +0 -39
  231. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_ops_on_diff_frames_groupby_expanding.py +0 -44
  232. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_ops_on_diff_frames_groupby_rolling.py +0 -84
  233. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_repr.py +0 -37
  234. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_resample.py +0 -45
  235. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_reshape.py +0 -39
  236. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_rolling.py +0 -39
  237. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_scalars.py +0 -37
  238. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_series_conversion.py +0 -39
  239. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_series_datetime.py +0 -39
  240. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_series_string.py +0 -39
  241. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_spark_functions.py +0 -39
  242. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_sql.py +0 -43
  243. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_stats.py +0 -37
  244. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_typedef.py +0 -36
  245. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_utils.py +0 -37
  246. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_window.py +0 -39
  247. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/__init__.py +0 -16
  248. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_base.py +0 -107
  249. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_binary_ops.py +0 -224
  250. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_boolean_ops.py +0 -825
  251. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_categorical_ops.py +0 -562
  252. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_complex_ops.py +0 -368
  253. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_date_ops.py +0 -257
  254. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_datetime_ops.py +0 -260
  255. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_null_ops.py +0 -178
  256. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_num_arithmetic.py +0 -184
  257. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_num_ops.py +0 -497
  258. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_num_reverse.py +0 -140
  259. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_string_ops.py +0 -354
  260. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_timedelta_ops.py +0 -219
  261. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_udt_ops.py +0 -192
  262. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/testing_utils.py +0 -228
  263. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/__init__.py +0 -16
  264. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_align.py +0 -118
  265. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_basic_slow.py +0 -198
  266. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_cov_corrwith.py +0 -181
  267. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_dot_frame.py +0 -103
  268. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_dot_series.py +0 -141
  269. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_index.py +0 -109
  270. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_series.py +0 -136
  271. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_setitem_frame.py +0 -125
  272. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_setitem_series.py +0 -217
  273. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/__init__.py +0 -16
  274. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_attrs.py +0 -384
  275. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_constructor.py +0 -598
  276. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_conversion.py +0 -73
  277. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_reindexing.py +0 -869
  278. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_reshaping.py +0 -487
  279. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_spark.py +0 -309
  280. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_take.py +0 -156
  281. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_time_series.py +0 -149
  282. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_truncate.py +0 -163
  283. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/__init__.py +0 -16
  284. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_aggregate.py +0 -311
  285. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_apply_func.py +0 -524
  286. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_cumulative.py +0 -419
  287. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_describe.py +0 -144
  288. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_groupby.py +0 -979
  289. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_head_tail.py +0 -234
  290. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_index.py +0 -206
  291. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_missing_data.py +0 -421
  292. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_split_apply.py +0 -187
  293. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_stat.py +0 -397
  294. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/__init__.py +0 -16
  295. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_align.py +0 -100
  296. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_base.py +0 -2743
  297. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_category.py +0 -484
  298. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_datetime.py +0 -276
  299. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_indexing.py +0 -432
  300. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_reindex.py +0 -310
  301. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_rename.py +0 -257
  302. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_reset_index.py +0 -160
  303. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_timedelta.py +0 -128
  304. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/io/__init__.py +0 -16
  305. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/io/test_io.py +0 -137
  306. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/plot/__init__.py +0 -16
  307. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/plot/test_frame_plot.py +0 -170
  308. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/plot/test_frame_plot_matplotlib.py +0 -547
  309. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/plot/test_frame_plot_plotly.py +0 -285
  310. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/plot/test_series_plot.py +0 -106
  311. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/plot/test_series_plot_matplotlib.py +0 -409
  312. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/plot/test_series_plot_plotly.py +0 -247
  313. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/__init__.py +0 -16
  314. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_all_any.py +0 -105
  315. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_arg_ops.py +0 -197
  316. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_as_of.py +0 -137
  317. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_as_type.py +0 -227
  318. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_compute.py +0 -634
  319. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_conversion.py +0 -88
  320. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_cumulative.py +0 -139
  321. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_index.py +0 -475
  322. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_missing_data.py +0 -265
  323. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_series.py +0 -818
  324. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_sort.py +0 -162
  325. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_stat.py +0 -780
  326. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_categorical.py +0 -741
  327. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_config.py +0 -160
  328. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_csv.py +0 -453
  329. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_dataframe_conversion.py +0 -281
  330. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_dataframe_spark_io.py +0 -487
  331. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_default_index.py +0 -109
  332. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_ewm.py +0 -434
  333. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_expanding.py +0 -253
  334. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_extension.py +0 -152
  335. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_frame_spark.py +0 -162
  336. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_generic_functions.py +0 -234
  337. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_indexing.py +0 -1339
  338. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_indexops_spark.py +0 -82
  339. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_internal.py +0 -124
  340. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_namespace.py +0 -638
  341. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_numpy_compat.py +0 -200
  342. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_ops_on_diff_frames.py +0 -1355
  343. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby.py +0 -655
  344. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_expanding.py +0 -113
  345. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_rolling.py +0 -118
  346. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_repr.py +0 -192
  347. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_resample.py +0 -346
  348. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_reshape.py +0 -495
  349. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_rolling.py +0 -263
  350. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_scalars.py +0 -59
  351. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_series_conversion.py +0 -85
  352. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_series_datetime.py +0 -364
  353. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_series_string.py +0 -362
  354. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_spark_functions.py +0 -46
  355. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_sql.py +0 -123
  356. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_stats.py +0 -581
  357. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_typedef.py +0 -447
  358. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_utils.py +0 -301
  359. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_window.py +0 -465
  360. snowflake/snowpark_connect/includes/python/pyspark/resource/tests/__init__.py +0 -16
  361. snowflake/snowpark_connect/includes/python/pyspark/resource/tests/test_resources.py +0 -83
  362. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/__init__.py +0 -16
  363. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/__init__.py +0 -16
  364. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/client/__init__.py +0 -16
  365. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/client/test_artifact.py +0 -420
  366. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/client/test_client.py +0 -358
  367. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/streaming/__init__.py +0 -16
  368. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/streaming/test_parity_foreach.py +0 -36
  369. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/streaming/test_parity_foreach_batch.py +0 -44
  370. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/streaming/test_parity_listener.py +0 -116
  371. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/streaming/test_parity_streaming.py +0 -35
  372. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_connect_basic.py +0 -3612
  373. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_connect_column.py +0 -1042
  374. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_connect_function.py +0 -2381
  375. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_connect_plan.py +0 -1060
  376. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_arrow.py +0 -163
  377. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_arrow_map.py +0 -38
  378. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_arrow_python_udf.py +0 -48
  379. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_catalog.py +0 -36
  380. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_column.py +0 -55
  381. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_conf.py +0 -36
  382. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_dataframe.py +0 -96
  383. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_datasources.py +0 -44
  384. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_errors.py +0 -36
  385. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_functions.py +0 -59
  386. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_group.py +0 -36
  387. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_pandas_cogrouped_map.py +0 -59
  388. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_pandas_grouped_map.py +0 -74
  389. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_pandas_grouped_map_with_state.py +0 -62
  390. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_pandas_map.py +0 -58
  391. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_pandas_udf.py +0 -70
  392. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_pandas_udf_grouped_agg.py +0 -50
  393. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_pandas_udf_scalar.py +0 -68
  394. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_pandas_udf_window.py +0 -40
  395. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_readwriter.py +0 -46
  396. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_serde.py +0 -44
  397. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_types.py +0 -100
  398. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_udf.py +0 -100
  399. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_udtf.py +0 -163
  400. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_session.py +0 -181
  401. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_utils.py +0 -42
  402. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/__init__.py +0 -16
  403. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py +0 -623
  404. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_grouped_map.py +0 -869
  405. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_grouped_map_with_state.py +0 -342
  406. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_map.py +0 -436
  407. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_udf.py +0 -363
  408. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_udf_grouped_agg.py +0 -592
  409. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_udf_scalar.py +0 -1503
  410. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_udf_typehints.py +0 -392
  411. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_udf_typehints_with_future_annotations.py +0 -375
  412. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_udf_window.py +0 -411
  413. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/streaming/__init__.py +0 -16
  414. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/streaming/test_streaming.py +0 -401
  415. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/streaming/test_streaming_foreach.py +0 -295
  416. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/streaming/test_streaming_foreach_batch.py +0 -106
  417. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/streaming/test_streaming_listener.py +0 -558
  418. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_arrow.py +0 -1346
  419. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_arrow_map.py +0 -182
  420. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_arrow_python_udf.py +0 -202
  421. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_catalog.py +0 -503
  422. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_column.py +0 -225
  423. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_conf.py +0 -83
  424. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_context.py +0 -201
  425. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_dataframe.py +0 -1931
  426. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_datasources.py +0 -256
  427. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_errors.py +0 -69
  428. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_functions.py +0 -1349
  429. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_group.py +0 -53
  430. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_pandas_sqlmetrics.py +0 -68
  431. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_readwriter.py +0 -283
  432. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_serde.py +0 -155
  433. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_session.py +0 -412
  434. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_types.py +0 -1581
  435. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_udf.py +0 -961
  436. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_udf_profiler.py +0 -165
  437. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_udtf.py +0 -1456
  438. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_utils.py +0 -1686
  439. snowflake/snowpark_connect/includes/python/pyspark/streaming/tests/__init__.py +0 -16
  440. snowflake/snowpark_connect/includes/python/pyspark/streaming/tests/test_context.py +0 -184
  441. snowflake/snowpark_connect/includes/python/pyspark/streaming/tests/test_dstream.py +0 -706
  442. snowflake/snowpark_connect/includes/python/pyspark/streaming/tests/test_kinesis.py +0 -118
  443. snowflake/snowpark_connect/includes/python/pyspark/streaming/tests/test_listener.py +0 -160
  444. snowflake/snowpark_connect/includes/python/pyspark/tests/__init__.py +0 -16
  445. snowflake/snowpark_connect/includes/python/pyspark/tests/test_appsubmit.py +0 -306
  446. snowflake/snowpark_connect/includes/python/pyspark/tests/test_broadcast.py +0 -196
  447. snowflake/snowpark_connect/includes/python/pyspark/tests/test_conf.py +0 -44
  448. snowflake/snowpark_connect/includes/python/pyspark/tests/test_context.py +0 -346
  449. snowflake/snowpark_connect/includes/python/pyspark/tests/test_daemon.py +0 -89
  450. snowflake/snowpark_connect/includes/python/pyspark/tests/test_install_spark.py +0 -124
  451. snowflake/snowpark_connect/includes/python/pyspark/tests/test_join.py +0 -69
  452. snowflake/snowpark_connect/includes/python/pyspark/tests/test_memory_profiler.py +0 -167
  453. snowflake/snowpark_connect/includes/python/pyspark/tests/test_pin_thread.py +0 -194
  454. snowflake/snowpark_connect/includes/python/pyspark/tests/test_profiler.py +0 -168
  455. snowflake/snowpark_connect/includes/python/pyspark/tests/test_rdd.py +0 -939
  456. snowflake/snowpark_connect/includes/python/pyspark/tests/test_rddbarrier.py +0 -52
  457. snowflake/snowpark_connect/includes/python/pyspark/tests/test_rddsampler.py +0 -66
  458. snowflake/snowpark_connect/includes/python/pyspark/tests/test_readwrite.py +0 -368
  459. snowflake/snowpark_connect/includes/python/pyspark/tests/test_serializers.py +0 -257
  460. snowflake/snowpark_connect/includes/python/pyspark/tests/test_shuffle.py +0 -267
  461. snowflake/snowpark_connect/includes/python/pyspark/tests/test_stage_sched.py +0 -153
  462. snowflake/snowpark_connect/includes/python/pyspark/tests/test_statcounter.py +0 -130
  463. snowflake/snowpark_connect/includes/python/pyspark/tests/test_taskcontext.py +0 -350
  464. snowflake/snowpark_connect/includes/python/pyspark/tests/test_util.py +0 -97
  465. snowflake/snowpark_connect/includes/python/pyspark/tests/test_worker.py +0 -271
  466. snowpark_connect-0.24.0.dist-info/RECORD +0 -898
  467. {snowpark_connect-0.24.0.data → snowpark_connect-0.25.0.data}/scripts/snowpark-connect +0 -0
  468. {snowpark_connect-0.24.0.data → snowpark_connect-0.25.0.data}/scripts/snowpark-session +0 -0
  469. {snowpark_connect-0.24.0.data → snowpark_connect-0.25.0.data}/scripts/snowpark-submit +0 -0
  470. {snowpark_connect-0.24.0.dist-info → snowpark_connect-0.25.0.dist-info}/WHEEL +0 -0
  471. {snowpark_connect-0.24.0.dist-info → snowpark_connect-0.25.0.dist-info}/licenses/LICENSE-binary +0 -0
  472. {snowpark_connect-0.24.0.dist-info → snowpark_connect-0.25.0.dist-info}/licenses/LICENSE.txt +0 -0
  473. {snowpark_connect-0.24.0.dist-info → snowpark_connect-0.25.0.dist-info}/licenses/NOTICE-binary +0 -0
  474. {snowpark_connect-0.24.0.dist-info → snowpark_connect-0.25.0.dist-info}/top_level.txt +0 -0
@@ -1,634 +0,0 @@
1
- #
2
- # Licensed to the Apache Software Foundation (ASF) under one or more
3
- # contributor license agreements. See the NOTICE file distributed with
4
- # this work for additional information regarding copyright ownership.
5
- # The ASF licenses this file to You under the Apache License, Version 2.0
6
- # (the "License"); you may not use this file except in compliance with
7
- # the License. You may obtain a copy of the License at
8
- #
9
- # http://www.apache.org/licenses/LICENSE-2.0
10
- #
11
- # Unless required by applicable law or agreed to in writing, software
12
- # distributed under the License is distributed on an "AS IS" BASIS,
13
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
- # See the License for the specific language governing permissions and
15
- # limitations under the License.
16
- #
17
- import unittest
18
- from distutils.version import LooseVersion
19
- from itertools import product
20
-
21
- import numpy as np
22
- import pandas as pd
23
-
24
- from pyspark import pandas as ps
25
- from pyspark.testing.pandasutils import ComparisonTestBase
26
- from pyspark.testing.sqlutils import SQLTestUtils
27
-
28
-
29
- class SeriesComputeMixin:
30
- @property
31
- def pser(self):
32
- return pd.Series([1, 2, 3, 4, 5, 6, 7], name="x")
33
-
34
- @property
35
- def psser(self):
36
- return ps.from_pandas(self.pser)
37
-
38
- def test_duplicated(self):
39
- for pser in [
40
- pd.Series(["beetle", None, "beetle", None, "lama", "beetle"], name="objects"),
41
- pd.Series([1, np.nan, 1, np.nan], name="numbers"),
42
- pd.Series(
43
- [
44
- pd.Timestamp("2022-01-01"),
45
- pd.Timestamp("2022-02-02"),
46
- pd.Timestamp("2022-01-01"),
47
- pd.Timestamp("2022-02-02"),
48
- ],
49
- name="times",
50
- ),
51
- ]:
52
- psser = ps.from_pandas(pser)
53
- self.assert_eq(psser.duplicated().sort_index(), pser.duplicated())
54
- self.assert_eq(
55
- psser.duplicated(keep="first").sort_index(), pser.duplicated(keep="first")
56
- )
57
- self.assert_eq(psser.duplicated(keep="last").sort_index(), pser.duplicated(keep="last"))
58
- self.assert_eq(psser.duplicated(keep=False).sort_index(), pser.duplicated(keep=False))
59
-
60
- pser = pd.Series([1, 2, 1, 2, 3], name="numbers")
61
- psser = ps.from_pandas(pser)
62
- self.assert_eq((psser + 1).duplicated().sort_index(), (pser + 1).duplicated())
63
-
64
- def test_drop_duplicates(self):
65
- pdf = pd.DataFrame({"animal": ["lama", "cow", "lama", "beetle", "lama", "hippo"]})
66
- psdf = ps.from_pandas(pdf)
67
-
68
- pser = pdf.animal
69
- psser = psdf.animal
70
-
71
- self.assert_eq(psser.drop_duplicates().sort_index(), pser.drop_duplicates().sort_index())
72
- self.assert_eq(
73
- psser.drop_duplicates(keep="last").sort_index(),
74
- pser.drop_duplicates(keep="last").sort_index(),
75
- )
76
-
77
- # inplace
78
- psser.drop_duplicates(keep=False, inplace=True)
79
- pser.drop_duplicates(keep=False, inplace=True)
80
- self.assert_eq(psser.sort_index(), pser.sort_index())
81
- self.assert_eq(psdf, pdf)
82
-
83
- def test_clip(self):
84
- pdf = pd.DataFrame({"x": [0, 2, 4]}, index=np.random.rand(3))
85
- psdf = ps.from_pandas(pdf)
86
- pser, psser = pdf.x, psdf.x
87
-
88
- # Assert list-like values are not accepted for 'lower' and 'upper'
89
- msg = "List-like value are not supported for 'lower' and 'upper' at the moment"
90
- with self.assertRaises(TypeError, msg=msg):
91
- psser.clip(lower=[1])
92
- with self.assertRaises(TypeError, msg=msg):
93
- psser.clip(upper=[1])
94
-
95
- # Assert no lower or upper
96
- self.assert_eq(psser.clip(), pser.clip())
97
- # Assert lower only
98
- self.assert_eq(psser.clip(1), pser.clip(1))
99
- # Assert upper only
100
- self.assert_eq(psser.clip(upper=3), pser.clip(upper=3))
101
- # Assert lower and upper
102
- self.assert_eq(psser.clip(1, 3), pser.clip(1, 3))
103
- self.assert_eq((psser + 1).clip(1, 3), (pser + 1).clip(1, 3))
104
-
105
- # Assert inplace is True
106
- pser.clip(1, 3, inplace=True)
107
- psser.clip(1, 3, inplace=True)
108
- self.assert_eq(psser, pser)
109
- self.assert_eq(psdf, pdf)
110
-
111
- # Assert behavior on string values
112
- str_psser = ps.Series(["a", "b", "c"])
113
- self.assert_eq(str_psser.clip(1, 3), str_psser)
114
-
115
- def test_compare(self):
116
- if LooseVersion(pd.__version__) >= LooseVersion("1.1"):
117
- pser = pd.Series([1, 2])
118
- psser = ps.from_pandas(pser)
119
-
120
- res_psdf = psser.compare(psser)
121
- self.assertTrue(res_psdf.empty)
122
- self.assert_eq(res_psdf.columns, pd.Index(["self", "other"]))
123
-
124
- self.assert_eq(
125
- pser.compare(pser + 1).sort_index(), psser.compare(psser + 1).sort_index()
126
- )
127
-
128
- pser = pd.Series([1, 2], index=["x", "y"])
129
- psser = ps.from_pandas(pser)
130
- self.assert_eq(
131
- pser.compare(pser + 1).sort_index(), psser.compare(psser + 1).sort_index()
132
- )
133
- else:
134
- psser = ps.Series([1, 2])
135
- res_psdf = psser.compare(psser)
136
- self.assertTrue(res_psdf.empty)
137
- self.assert_eq(res_psdf.columns, pd.Index(["self", "other"]))
138
- expected = ps.DataFrame([[1, 2], [2, 3]], columns=["self", "other"])
139
- self.assert_eq(expected, psser.compare(psser + 1).sort_index())
140
-
141
- psser = ps.Series([1, 2], index=["x", "y"])
142
- expected = ps.DataFrame([[1, 2], [2, 3]], index=["x", "y"], columns=["self", "other"])
143
- self.assert_eq(expected, psser.compare(psser + 1).sort_index())
144
-
145
- @unittest.skipIf(
146
- LooseVersion(pd.__version__) >= LooseVersion("2.0.0"),
147
- "TODO(SPARK-43465): Enable SeriesTests.test_append for pandas 2.0.0.",
148
- )
149
- def test_append(self):
150
- pser1 = pd.Series([1, 2, 3], name="0")
151
- pser2 = pd.Series([4, 5, 6], name="0")
152
- pser3 = pd.Series([4, 5, 6], index=[3, 4, 5], name="0")
153
- psser1 = ps.from_pandas(pser1)
154
- psser2 = ps.from_pandas(pser2)
155
- psser3 = ps.from_pandas(pser3)
156
-
157
- self.assert_eq(psser1.append(psser2), pser1.append(pser2))
158
- self.assert_eq(psser1.append(psser3), pser1.append(pser3))
159
- self.assert_eq(
160
- psser1.append(psser2, ignore_index=True), pser1.append(pser2, ignore_index=True)
161
- )
162
-
163
- psser1.append(psser3, verify_integrity=True)
164
- msg = "Indices have overlapping values"
165
- with self.assertRaises(ValueError, msg=msg):
166
- psser1.append(psser2, verify_integrity=True)
167
-
168
- def test_shift(self):
169
- pser = pd.Series([10, 20, 15, 30, 45], name="x")
170
- psser = ps.Series(pser)
171
-
172
- self.assert_eq(psser.shift(2), pser.shift(2))
173
- self.assert_eq(psser.shift().shift(-1), pser.shift().shift(-1))
174
- self.assert_eq(psser.shift().sum(), pser.shift().sum())
175
-
176
- self.assert_eq(psser.shift(periods=2, fill_value=0), pser.shift(periods=2, fill_value=0))
177
-
178
- with self.assertRaisesRegex(TypeError, "periods should be an int; however"):
179
- psser.shift(periods=1.5)
180
-
181
- self.assert_eq(psser.shift(periods=0), pser.shift(periods=0))
182
-
183
- def test_diff(self):
184
- pser = pd.Series([10, 20, 15, 30, 45], name="x")
185
- psser = ps.Series(pser)
186
-
187
- self.assert_eq(psser.diff(2), pser.diff(2))
188
- self.assert_eq(psser.diff().diff(-1), pser.diff().diff(-1))
189
- self.assert_eq(psser.diff().sum(), pser.diff().sum())
190
-
191
- def test_aggregate(self):
192
- pser = pd.Series([10, 20, 15, 30, 45], name="x")
193
- psser = ps.Series(pser)
194
- msg = "func must be a string or list of strings"
195
- with self.assertRaisesRegex(TypeError, msg):
196
- psser.aggregate({"x": ["min", "max"]})
197
- msg = (
198
- "If the given function is a list, it " "should only contains function names as strings."
199
- )
200
- with self.assertRaisesRegex(ValueError, msg):
201
- psser.aggregate(["min", max])
202
-
203
- def test_drop(self):
204
- pdf = pd.DataFrame({"x": [10, 20, 15, 30, 45]})
205
- psdf = ps.from_pandas(pdf)
206
- pser, psser = pdf.x, psdf.x
207
-
208
- self.assert_eq(psser.drop(1), pser.drop(1))
209
- self.assert_eq(psser.drop([1, 4]), pser.drop([1, 4]))
210
- self.assert_eq(psser.drop(columns=1), pser.drop(columns=1))
211
- self.assert_eq(psser.drop(columns=[1, 4]), pser.drop(columns=[1, 4]))
212
-
213
- msg = "Need to specify at least one of 'labels', 'index' or 'columns'"
214
- with self.assertRaisesRegex(ValueError, msg):
215
- psser.drop()
216
- self.assertRaises(KeyError, lambda: psser.drop((0, 1)))
217
-
218
- psser.drop([2, 3], inplace=True)
219
- pser.drop([2, 3], inplace=True)
220
- self.assert_eq(psser, pser)
221
- self.assert_eq(psdf, pdf)
222
-
223
- n_pser, n_psser = pser + 1, psser + 1
224
- n_psser.drop([1, 4], inplace=True)
225
- n_pser.drop([1, 4], inplace=True)
226
- self.assert_eq(n_psser, n_pser)
227
- self.assert_eq(psser, pser)
228
-
229
- # For MultiIndex
230
- midx = pd.MultiIndex(
231
- [["lama", "cow", "falcon"], ["speed", "weight", "length"]],
232
- [[0, 0, 0, 1, 1, 1, 2, 2, 2], [0, 1, 2, 0, 1, 2, 0, 1, 2]],
233
- )
234
-
235
- pdf = pd.DataFrame({"x": [45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3]}, index=midx)
236
- psdf = ps.from_pandas(pdf)
237
- psser, pser = psdf.x, pdf.x
238
-
239
- self.assert_eq(psser.drop("lama"), pser.drop("lama"))
240
- self.assert_eq(psser.drop(labels="weight", level=1), pser.drop(labels="weight", level=1))
241
- self.assert_eq(psser.drop(("lama", "weight")), pser.drop(("lama", "weight")))
242
- self.assert_eq(
243
- psser.drop([("lama", "speed"), ("falcon", "weight")]),
244
- pser.drop([("lama", "speed"), ("falcon", "weight")]),
245
- )
246
- self.assert_eq(psser.drop({"lama": "speed"}), pser.drop({"lama": "speed"}))
247
-
248
- msg = "'level' should be less than the number of indexes"
249
- with self.assertRaisesRegex(ValueError, msg):
250
- psser.drop(labels="weight", level=2)
251
-
252
- msg = (
253
- "If the given index is a list, it "
254
- "should only contains names as all tuples or all non tuples "
255
- "that contain index names"
256
- )
257
- with self.assertRaisesRegex(ValueError, msg):
258
- psser.drop(["lama", ["cow", "falcon"]])
259
-
260
- msg = "Cannot specify both 'labels' and 'index'/'columns'"
261
- with self.assertRaisesRegex(ValueError, msg):
262
- psser.drop("lama", index="cow")
263
-
264
- with self.assertRaisesRegex(ValueError, msg):
265
- psser.drop("lama", columns="cow")
266
-
267
- msg = r"'Key length \(2\) exceeds index depth \(3\)'"
268
- with self.assertRaisesRegex(KeyError, msg):
269
- psser.drop(("lama", "speed", "x"))
270
-
271
- psser.drop({"lama": "speed"}, inplace=True)
272
- pser.drop({"lama": "speed"}, inplace=True)
273
- self.assert_eq(psser, pser)
274
- self.assert_eq(psdf, pdf)
275
-
276
- def test_pop(self):
277
- midx = pd.MultiIndex(
278
- [["lama", "cow", "falcon"], ["speed", "weight", "length"]],
279
- [[0, 0, 0, 1, 1, 1, 2, 2, 2], [0, 1, 2, 0, 1, 2, 0, 1, 2]],
280
- )
281
- pdf = pd.DataFrame({"x": [45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3]}, index=midx)
282
- psdf = ps.from_pandas(pdf)
283
-
284
- pser = pdf.x
285
- psser = psdf.x
286
-
287
- self.assert_eq(psser.pop(("lama", "speed")), pser.pop(("lama", "speed")))
288
- self.assert_eq(psser, pser)
289
- self.assert_eq(psdf, pdf)
290
-
291
- msg = r"'Key length \(3\) exceeds index depth \(2\)'"
292
- with self.assertRaisesRegex(KeyError, msg):
293
- psser.pop(("lama", "speed", "x"))
294
-
295
- msg = "'key' should be string or tuple that contains strings"
296
- with self.assertRaisesRegex(TypeError, msg):
297
- psser.pop(["lama", "speed"])
298
-
299
- pser = pd.Series(["a", "b", "c", "a"], dtype="category")
300
- psser = ps.from_pandas(pser)
301
-
302
- if LooseVersion(pd.__version__) >= LooseVersion("1.3.0"):
303
- self.assert_eq(psser.pop(0), pser.pop(0))
304
- self.assert_eq(psser, pser)
305
-
306
- self.assert_eq(psser.pop(3), pser.pop(3))
307
- self.assert_eq(psser, pser)
308
- else:
309
- # Before pandas 1.3.0, `pop` modifies the dtype of categorical series wrongly.
310
- self.assert_eq(psser.pop(0), "a")
311
- self.assert_eq(
312
- psser,
313
- pd.Series(
314
- pd.Categorical(["b", "c", "a"], categories=["a", "b", "c"]), index=[1, 2, 3]
315
- ),
316
- )
317
-
318
- self.assert_eq(psser.pop(3), "a")
319
- self.assert_eq(
320
- psser,
321
- pd.Series(pd.Categorical(["b", "c"], categories=["a", "b", "c"]), index=[1, 2]),
322
- )
323
-
324
- def test_duplicates(self):
325
- psers = {
326
- "test on texts": pd.Series(
327
- ["lama", "cow", "lama", "beetle", "lama", "hippo"], name="animal"
328
- ),
329
- "test on numbers": pd.Series([1, 1, 2, 4, 3]),
330
- }
331
- keeps = ["first", "last", False]
332
-
333
- for (msg, pser), keep in product(psers.items(), keeps):
334
- with self.subTest(msg, keep=keep):
335
- psser = ps.Series(pser)
336
-
337
- self.assert_eq(
338
- pser.drop_duplicates(keep=keep).sort_values(),
339
- psser.drop_duplicates(keep=keep).sort_values(),
340
- )
341
-
342
- def test_truncate(self):
343
- pser1 = pd.Series([10, 20, 30, 40, 50, 60, 70], index=[1, 2, 3, 4, 5, 6, 7])
344
- psser1 = ps.Series(pser1)
345
- pser2 = pd.Series([10, 20, 30, 40, 50, 60, 70], index=[7, 6, 5, 4, 3, 2, 1])
346
- psser2 = ps.Series(pser2)
347
-
348
- self.assert_eq(psser1.truncate(), pser1.truncate())
349
- self.assert_eq(psser1.truncate(before=2), pser1.truncate(before=2))
350
- self.assert_eq(psser1.truncate(after=5), pser1.truncate(after=5))
351
- self.assert_eq(psser1.truncate(copy=False), pser1.truncate(copy=False))
352
- self.assert_eq(psser1.truncate(2, 5, copy=False), pser1.truncate(2, 5, copy=False))
353
- # The bug for these tests has been fixed in pandas 1.1.0.
354
- if LooseVersion(pd.__version__) >= LooseVersion("1.1.0"):
355
- self.assert_eq(psser2.truncate(4, 6), pser2.truncate(4, 6))
356
- self.assert_eq(psser2.truncate(4, 6, copy=False), pser2.truncate(4, 6, copy=False))
357
- else:
358
- expected_psser = ps.Series([20, 30, 40], index=[6, 5, 4])
359
- self.assert_eq(psser2.truncate(4, 6), expected_psser)
360
- self.assert_eq(psser2.truncate(4, 6, copy=False), expected_psser)
361
-
362
- psser = ps.Series([10, 20, 30, 40, 50, 60, 70], index=[1, 2, 3, 4, 3, 2, 1])
363
- msg = "truncate requires a sorted index"
364
- with self.assertRaisesRegex(ValueError, msg):
365
- psser.truncate()
366
-
367
- psser = ps.Series([10, 20, 30, 40, 50, 60, 70], index=[1, 2, 3, 4, 5, 6, 7])
368
- msg = "Truncate: 2 must be after 5"
369
- with self.assertRaisesRegex(ValueError, msg):
370
- psser.truncate(5, 2)
371
-
372
- def test_unstack(self):
373
- pser = pd.Series(
374
- [10, -2, 4, 7],
375
- index=pd.MultiIndex.from_tuples(
376
- [("one", "a", "z"), ("one", "b", "x"), ("two", "a", "c"), ("two", "b", "v")],
377
- names=["A", "B", "C"],
378
- ),
379
- )
380
- psser = ps.from_pandas(pser)
381
-
382
- levels = [-3, -2, -1, 0, 1, 2]
383
- for level in levels:
384
- pandas_result = pser.unstack(level=level)
385
- pandas_on_spark_result = psser.unstack(level=level).sort_index()
386
- self.assert_eq(pandas_result, pandas_on_spark_result)
387
- self.assert_eq(pandas_result.index.names, pandas_on_spark_result.index.names)
388
- self.assert_eq(pandas_result.columns.names, pandas_on_spark_result.columns.names)
389
-
390
- # non-numeric datatypes
391
- pser = pd.Series(
392
- list("abcd"), index=pd.MultiIndex.from_product([["one", "two"], ["a", "b"]])
393
- )
394
- psser = ps.from_pandas(pser)
395
-
396
- levels = [-2, -1, 0, 1]
397
- for level in levels:
398
- pandas_result = pser.unstack(level=level)
399
- pandas_on_spark_result = psser.unstack(level=level).sort_index()
400
- self.assert_eq(pandas_result, pandas_on_spark_result)
401
- self.assert_eq(pandas_result.index.names, pandas_on_spark_result.index.names)
402
- self.assert_eq(pandas_result.columns.names, pandas_on_spark_result.columns.names)
403
-
404
- # Exceeding the range of level
405
- self.assertRaises(IndexError, lambda: psser.unstack(level=3))
406
- self.assertRaises(IndexError, lambda: psser.unstack(level=-4))
407
- # Only support for MultiIndex
408
- psser = ps.Series([10, -2, 4, 7])
409
- self.assertRaises(ValueError, lambda: psser.unstack())
410
-
411
- def test_abs(self):
412
- pser = pd.Series([-2, -1, 0, 1])
413
- psser = ps.from_pandas(pser)
414
-
415
- self.assert_eq(abs(psser), abs(pser))
416
- self.assert_eq(np.abs(psser), np.abs(pser))
417
-
418
- @unittest.skipIf(
419
- LooseVersion(pd.__version__) >= LooseVersion("2.0.0"),
420
- "TODO(SPARK-43550): Enable SeriesTests.test_factorize for pandas 2.0.0.",
421
- )
422
- def test_factorize(self):
423
- pser = pd.Series(["a", "b", "a", "b"])
424
- psser = ps.from_pandas(pser)
425
- pcodes, puniques = pser.factorize(sort=True)
426
- kcodes, kuniques = psser.factorize()
427
- self.assert_eq(pcodes.tolist(), kcodes.to_list())
428
- self.assert_eq(puniques, kuniques)
429
-
430
- pser = pd.Series([5, 1, 5, 1])
431
- psser = ps.from_pandas(pser)
432
- pcodes, puniques = (pser + 1).factorize(sort=True)
433
- kcodes, kuniques = (psser + 1).factorize()
434
- self.assert_eq(pcodes.tolist(), kcodes.to_list())
435
- self.assert_eq(puniques, kuniques)
436
-
437
- pser = pd.Series(["a", "b", "a", "b"], name="ser", index=["w", "x", "y", "z"])
438
- psser = ps.from_pandas(pser)
439
- pcodes, puniques = pser.factorize(sort=True)
440
- kcodes, kuniques = psser.factorize()
441
- self.assert_eq(pcodes.tolist(), kcodes.to_list())
442
- self.assert_eq(puniques, kuniques)
443
-
444
- pser = pd.Series(
445
- ["a", "b", "a", "b"], index=pd.MultiIndex.from_arrays([[4, 3, 2, 1], [1, 2, 3, 4]])
446
- )
447
- psser = ps.from_pandas(pser)
448
- pcodes, puniques = pser.factorize(sort=True)
449
- kcodes, kuniques = psser.factorize()
450
- self.assert_eq(pcodes.tolist(), kcodes.to_list())
451
- self.assert_eq(puniques, kuniques)
452
-
453
- #
454
- # Deals with None and np.nan
455
- #
456
- pser = pd.Series(["a", "b", "a", np.nan])
457
- psser = ps.from_pandas(pser)
458
- pcodes, puniques = pser.factorize(sort=True)
459
- kcodes, kuniques = psser.factorize()
460
- self.assert_eq(pcodes.tolist(), kcodes.to_list())
461
- self.assert_eq(puniques, kuniques)
462
-
463
- pser = pd.Series([1, None, 3, 2, 1])
464
- psser = ps.from_pandas(pser)
465
- pcodes, puniques = pser.factorize(sort=True)
466
- kcodes, kuniques = psser.factorize()
467
- self.assert_eq(pcodes.tolist(), kcodes.to_list())
468
- self.assert_eq(puniques, kuniques)
469
-
470
- pser = pd.Series(["a", None, "a"])
471
- psser = ps.from_pandas(pser)
472
- pcodes, puniques = pser.factorize(sort=True)
473
- kcodes, kuniques = psser.factorize()
474
- self.assert_eq(pcodes.tolist(), kcodes.to_list())
475
- self.assert_eq(puniques, kuniques)
476
-
477
- pser = pd.Series([None, np.nan])
478
- psser = ps.from_pandas(pser)
479
- pcodes, puniques = pser.factorize()
480
- kcodes, kuniques = psser.factorize()
481
- self.assert_eq(pcodes, kcodes.to_list())
482
- # pandas: Float64Index([], dtype='float64')
483
- self.assert_eq(pd.Index([]), kuniques)
484
-
485
- pser = pd.Series([np.nan, np.nan])
486
- psser = ps.from_pandas(pser)
487
- pcodes, puniques = pser.factorize()
488
- kcodes, kuniques = psser.factorize()
489
- self.assert_eq(pcodes, kcodes.to_list())
490
- # pandas: Float64Index([], dtype='float64')
491
- self.assert_eq(pd.Index([]), kuniques)
492
-
493
- #
494
- # Deals with na_sentinel
495
- #
496
- # pandas >= 1.1.2 support na_sentinel=None
497
- #
498
- pd_below_1_1_2 = LooseVersion(pd.__version__) < LooseVersion("1.1.2")
499
-
500
- pser = pd.Series(["a", "b", "a", np.nan, None])
501
- psser = ps.from_pandas(pser)
502
-
503
- pcodes, puniques = pser.factorize(sort=True, na_sentinel=-2)
504
- kcodes, kuniques = psser.factorize(na_sentinel=-2)
505
- self.assert_eq(pcodes.tolist(), kcodes.to_list())
506
- self.assert_eq(puniques, kuniques)
507
-
508
- pcodes, puniques = pser.factorize(sort=True, na_sentinel=2)
509
- kcodes, kuniques = psser.factorize(na_sentinel=2)
510
- self.assert_eq(pcodes.tolist(), kcodes.to_list())
511
- self.assert_eq(puniques, kuniques)
512
-
513
- if not pd_below_1_1_2:
514
- pcodes, puniques = pser.factorize(sort=True, na_sentinel=None)
515
- kcodes, kuniques = psser.factorize(na_sentinel=None)
516
- self.assert_eq(pcodes.tolist(), kcodes.to_list())
517
- # puniques is Index(['a', 'b', nan], dtype='object')
518
- self.assert_eq(ps.Index(["a", "b", None]), kuniques)
519
-
520
- psser = ps.Series([1, 2, np.nan, 4, 5]) # Arrow takes np.nan as null
521
- psser.loc[3] = np.nan # Spark takes np.nan as NaN
522
- kcodes, kuniques = psser.factorize(na_sentinel=None)
523
- pcodes, puniques = psser._to_pandas().factorize(sort=True, na_sentinel=None)
524
- self.assert_eq(pcodes.tolist(), kcodes.to_list())
525
- self.assert_eq(puniques, kuniques)
526
-
527
- def test_explode(self):
528
- pser = pd.Series([[1, 2, 3], [], None, [3, 4]])
529
- psser = ps.from_pandas(pser)
530
- self.assert_eq(pser.explode(), psser.explode(), almost=True)
531
-
532
- # MultiIndex
533
- pser.index = pd.MultiIndex.from_tuples([("a", "w"), ("b", "x"), ("c", "y"), ("d", "z")])
534
- psser = ps.from_pandas(pser)
535
- self.assert_eq(pser.explode(), psser.explode(), almost=True)
536
-
537
- # non-array type Series
538
- pser = pd.Series([1, 2, 3, 4])
539
- psser = ps.from_pandas(pser)
540
- self.assert_eq(pser.explode(), psser.explode())
541
-
542
- @unittest.skipIf(
543
- LooseVersion(pd.__version__) >= LooseVersion("2.0.0"),
544
- "TODO(SPARK-43467): Enable SeriesTests.test_between for pandas 2.0.0.",
545
- )
546
- def test_between(self):
547
- pser = pd.Series([np.nan, 1, 2, 3, 4])
548
- psser = ps.from_pandas(pser)
549
- self.assert_eq(psser.between(1, 4), pser.between(1, 4))
550
- self.assert_eq(psser.between(1, 4, inclusive="both"), pser.between(1, 4, inclusive="both"))
551
- self.assert_eq(
552
- psser.between(1, 4, inclusive="neither"), pser.between(1, 4, inclusive="neither")
553
- )
554
- self.assert_eq(psser.between(1, 4, inclusive="left"), pser.between(1, 4, inclusive="left"))
555
- self.assert_eq(
556
- psser.between(1, 4, inclusive="right"), pser.between(1, 4, inclusive="right")
557
- )
558
- expected_err_msg = (
559
- "Inclusive has to be either string of 'both'," "'left', 'right', or 'neither'"
560
- )
561
- with self.assertRaisesRegex(ValueError, expected_err_msg):
562
- psser.between(1, 4, inclusive="middle")
563
-
564
- # Test for backward compatibility
565
- self.assert_eq(psser.between(1, 4, inclusive=True), pser.between(1, 4, inclusive=True))
566
- self.assert_eq(psser.between(1, 4, inclusive=False), pser.between(1, 4, inclusive=False))
567
- with self.assertWarns(FutureWarning):
568
- psser.between(1, 4, inclusive=True)
569
-
570
- @unittest.skipIf(
571
- LooseVersion(pd.__version__) >= LooseVersion("2.0.0"),
572
- "TODO(SPARK-43479): Enable SeriesTests.test_between_time for pandas 2.0.0.",
573
- )
574
- def test_between_time(self):
575
- idx = pd.date_range("2018-04-09", periods=4, freq="1D20min")
576
- pser = pd.Series([1, 2, 3, 4], index=idx)
577
- psser = ps.from_pandas(pser)
578
- self.assert_eq(
579
- pser.between_time("0:15", "0:45").sort_index(),
580
- psser.between_time("0:15", "0:45").sort_index(),
581
- )
582
-
583
- pser.index.name = "ts"
584
- psser = ps.from_pandas(pser)
585
- self.assert_eq(
586
- pser.between_time("0:15", "0:45").sort_index(),
587
- psser.between_time("0:15", "0:45").sort_index(),
588
- )
589
-
590
- pser.index.name = "index"
591
- psser = ps.from_pandas(pser)
592
- self.assert_eq(
593
- pser.between_time("0:15", "0:45").sort_index(),
594
- psser.between_time("0:15", "0:45").sort_index(),
595
- )
596
-
597
- def test_at_time(self):
598
- idx = pd.date_range("2018-04-09", periods=4, freq="1D20min")
599
- pser = pd.Series([1, 2, 3, 4], index=idx)
600
- psser = ps.from_pandas(pser)
601
- self.assert_eq(
602
- pser.at_time("0:20").sort_index(),
603
- psser.at_time("0:20").sort_index(),
604
- )
605
-
606
- pser.index.name = "ts"
607
- psser = ps.from_pandas(pser)
608
- self.assert_eq(
609
- pser.at_time("0:20").sort_index(),
610
- psser.at_time("0:20").sort_index(),
611
- )
612
-
613
- pser.index.name = "index"
614
- psser = ps.from_pandas(pser)
615
- self.assert_eq(
616
- pser.at_time("0:20").sort_index(),
617
- psser.at_time("0:20").sort_index(),
618
- )
619
-
620
-
621
- class SeriesComputeTests(SeriesComputeMixin, ComparisonTestBase, SQLTestUtils):
622
- pass
623
-
624
-
625
- if __name__ == "__main__":
626
- from pyspark.pandas.tests.series.test_compute import * # noqa: F401
627
-
628
- try:
629
- import xmlrunner
630
-
631
- testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
632
- except ImportError:
633
- testRunner = None
634
- unittest.main(testRunner=testRunner, verbosity=2)