snowpark-connect 0.24.0__py3-none-any.whl → 0.25.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of snowpark-connect might be problematic. Click here for more details.

Files changed (474) hide show
  1. snowflake/snowpark_connect/column_name_handler.py +116 -4
  2. snowflake/snowpark_connect/config.py +13 -0
  3. snowflake/snowpark_connect/constants.py +0 -29
  4. snowflake/snowpark_connect/dataframe_container.py +6 -0
  5. snowflake/snowpark_connect/execute_plan/map_execution_command.py +56 -1
  6. snowflake/snowpark_connect/expression/literal.py +13 -2
  7. snowflake/snowpark_connect/expression/map_cast.py +5 -8
  8. snowflake/snowpark_connect/expression/map_sql_expression.py +23 -1
  9. snowflake/snowpark_connect/expression/map_udf.py +26 -8
  10. snowflake/snowpark_connect/expression/map_unresolved_attribute.py +199 -15
  11. snowflake/snowpark_connect/expression/map_unresolved_extract_value.py +44 -16
  12. snowflake/snowpark_connect/expression/map_unresolved_function.py +825 -353
  13. snowflake/snowpark_connect/expression/map_unresolved_star.py +3 -2
  14. snowflake/snowpark_connect/hidden_column.py +39 -0
  15. snowflake/snowpark_connect/includes/jars/hadoop-client-api-trimmed-3.3.4.jar +0 -0
  16. snowflake/snowpark_connect/includes/jars/{hadoop-client-api-3.3.4.jar → spark-connect-client-jvm_2.12-3.5.6.jar} +0 -0
  17. snowflake/snowpark_connect/relation/map_column_ops.py +17 -4
  18. snowflake/snowpark_connect/relation/map_extension.py +52 -11
  19. snowflake/snowpark_connect/relation/map_join.py +258 -62
  20. snowflake/snowpark_connect/relation/map_sql.py +88 -11
  21. snowflake/snowpark_connect/relation/map_udtf.py +4 -2
  22. snowflake/snowpark_connect/relation/read/map_read.py +3 -3
  23. snowflake/snowpark_connect/relation/read/map_read_jdbc.py +1 -1
  24. snowflake/snowpark_connect/relation/read/map_read_json.py +8 -1
  25. snowflake/snowpark_connect/relation/read/map_read_table.py +1 -9
  26. snowflake/snowpark_connect/relation/read/reader_config.py +3 -1
  27. snowflake/snowpark_connect/relation/write/map_write.py +62 -53
  28. snowflake/snowpark_connect/resources_initializer.py +29 -1
  29. snowflake/snowpark_connect/server.py +18 -3
  30. snowflake/snowpark_connect/type_mapping.py +29 -25
  31. snowflake/snowpark_connect/typed_column.py +14 -0
  32. snowflake/snowpark_connect/utils/artifacts.py +23 -0
  33. snowflake/snowpark_connect/utils/context.py +6 -1
  34. snowflake/snowpark_connect/utils/scala_udf_utils.py +588 -0
  35. snowflake/snowpark_connect/utils/telemetry.py +6 -17
  36. snowflake/snowpark_connect/utils/udf_helper.py +2 -0
  37. snowflake/snowpark_connect/utils/udf_utils.py +38 -7
  38. snowflake/snowpark_connect/utils/udtf_utils.py +17 -3
  39. snowflake/snowpark_connect/version.py +1 -1
  40. {snowpark_connect-0.24.0.dist-info → snowpark_connect-0.25.0.dist-info}/METADATA +1 -1
  41. snowpark_connect-0.25.0.dist-info/RECORD +477 -0
  42. snowflake/snowpark_connect/includes/jars/scala-compiler-2.12.18.jar +0 -0
  43. snowflake/snowpark_connect/includes/jars/spark-kubernetes_2.12-3.5.6.jar +0 -0
  44. snowflake/snowpark_connect/includes/jars/spark-mllib_2.12-3.5.6.jar +0 -0
  45. snowflake/snowpark_connect/includes/jars/spark-streaming_2.12-3.5.6.jar +0 -0
  46. snowflake/snowpark_connect/includes/python/pyspark/errors/tests/__init__.py +0 -16
  47. snowflake/snowpark_connect/includes/python/pyspark/errors/tests/test_errors.py +0 -60
  48. snowflake/snowpark_connect/includes/python/pyspark/ml/deepspeed/tests/test_deepspeed_distributor.py +0 -306
  49. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/__init__.py +0 -16
  50. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_connect_classification.py +0 -53
  51. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_connect_evaluation.py +0 -50
  52. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_connect_feature.py +0 -43
  53. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_connect_function.py +0 -114
  54. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_connect_pipeline.py +0 -47
  55. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_connect_summarizer.py +0 -43
  56. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_connect_tuning.py +0 -46
  57. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_legacy_mode_classification.py +0 -238
  58. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_legacy_mode_evaluation.py +0 -194
  59. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_legacy_mode_feature.py +0 -156
  60. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_legacy_mode_pipeline.py +0 -184
  61. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_legacy_mode_summarizer.py +0 -78
  62. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_legacy_mode_tuning.py +0 -292
  63. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_parity_torch_data_loader.py +0 -50
  64. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_parity_torch_distributor.py +0 -152
  65. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_algorithms.py +0 -456
  66. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_base.py +0 -96
  67. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_dl_util.py +0 -186
  68. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_evaluation.py +0 -77
  69. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_feature.py +0 -401
  70. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_functions.py +0 -528
  71. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_image.py +0 -82
  72. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_linalg.py +0 -409
  73. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_model_cache.py +0 -55
  74. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_param.py +0 -441
  75. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_persistence.py +0 -546
  76. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_pipeline.py +0 -71
  77. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_stat.py +0 -52
  78. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_training_summary.py +0 -494
  79. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_util.py +0 -85
  80. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_wrapper.py +0 -138
  81. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/__init__.py +0 -16
  82. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/test_cv_io_basic.py +0 -151
  83. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/test_cv_io_nested.py +0 -97
  84. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/test_cv_io_pipeline.py +0 -143
  85. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/test_tuning.py +0 -551
  86. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/test_tvs_io_basic.py +0 -137
  87. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/test_tvs_io_nested.py +0 -96
  88. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/test_tvs_io_pipeline.py +0 -142
  89. snowflake/snowpark_connect/includes/python/pyspark/ml/torch/tests/__init__.py +0 -16
  90. snowflake/snowpark_connect/includes/python/pyspark/ml/torch/tests/test_data_loader.py +0 -137
  91. snowflake/snowpark_connect/includes/python/pyspark/ml/torch/tests/test_distributor.py +0 -561
  92. snowflake/snowpark_connect/includes/python/pyspark/ml/torch/tests/test_log_communication.py +0 -172
  93. snowflake/snowpark_connect/includes/python/pyspark/mllib/tests/__init__.py +0 -16
  94. snowflake/snowpark_connect/includes/python/pyspark/mllib/tests/test_algorithms.py +0 -353
  95. snowflake/snowpark_connect/includes/python/pyspark/mllib/tests/test_feature.py +0 -192
  96. snowflake/snowpark_connect/includes/python/pyspark/mllib/tests/test_linalg.py +0 -680
  97. snowflake/snowpark_connect/includes/python/pyspark/mllib/tests/test_stat.py +0 -206
  98. snowflake/snowpark_connect/includes/python/pyspark/mllib/tests/test_streaming_algorithms.py +0 -471
  99. snowflake/snowpark_connect/includes/python/pyspark/mllib/tests/test_util.py +0 -108
  100. snowflake/snowpark_connect/includes/python/pyspark/pandas/spark/__init__.py +0 -16
  101. snowflake/snowpark_connect/includes/python/pyspark/pandas/spark/accessors.py +0 -1281
  102. snowflake/snowpark_connect/includes/python/pyspark/pandas/spark/functions.py +0 -203
  103. snowflake/snowpark_connect/includes/python/pyspark/pandas/spark/utils.py +0 -202
  104. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/__init__.py +0 -16
  105. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/__init__.py +0 -16
  106. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_any_all.py +0 -177
  107. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_apply_func.py +0 -575
  108. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_binary_ops.py +0 -235
  109. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_combine.py +0 -653
  110. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_compute.py +0 -463
  111. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_corrwith.py +0 -86
  112. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_cov.py +0 -151
  113. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_cumulative.py +0 -139
  114. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_describe.py +0 -458
  115. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_eval.py +0 -86
  116. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_melt.py +0 -202
  117. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_missing_data.py +0 -520
  118. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_pivot.py +0 -361
  119. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/__init__.py +0 -16
  120. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/__init__.py +0 -16
  121. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_any_all.py +0 -40
  122. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_apply_func.py +0 -42
  123. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_binary_ops.py +0 -40
  124. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_combine.py +0 -37
  125. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_compute.py +0 -60
  126. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_corrwith.py +0 -40
  127. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_cov.py +0 -40
  128. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_cumulative.py +0 -90
  129. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_describe.py +0 -40
  130. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_eval.py +0 -40
  131. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_melt.py +0 -40
  132. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_missing_data.py +0 -42
  133. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_pivot.py +0 -37
  134. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/__init__.py +0 -16
  135. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_base.py +0 -36
  136. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_binary_ops.py +0 -42
  137. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_boolean_ops.py +0 -47
  138. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_categorical_ops.py +0 -55
  139. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_complex_ops.py +0 -40
  140. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_date_ops.py +0 -47
  141. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_datetime_ops.py +0 -47
  142. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_null_ops.py +0 -42
  143. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_num_arithmetic.py +0 -43
  144. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_num_ops.py +0 -47
  145. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_num_reverse.py +0 -43
  146. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_string_ops.py +0 -47
  147. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_timedelta_ops.py +0 -47
  148. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_udt_ops.py +0 -40
  149. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/testing_utils.py +0 -226
  150. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/__init__.py +0 -16
  151. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_align.py +0 -39
  152. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_basic_slow.py +0 -55
  153. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_cov_corrwith.py +0 -39
  154. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_dot_frame.py +0 -39
  155. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_dot_series.py +0 -39
  156. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_index.py +0 -39
  157. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_series.py +0 -39
  158. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_setitem_frame.py +0 -43
  159. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_setitem_series.py +0 -43
  160. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/__init__.py +0 -16
  161. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_attrs.py +0 -40
  162. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_constructor.py +0 -39
  163. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_conversion.py +0 -42
  164. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_reindexing.py +0 -42
  165. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_reshaping.py +0 -37
  166. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_spark.py +0 -40
  167. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_take.py +0 -42
  168. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_time_series.py +0 -48
  169. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_truncate.py +0 -40
  170. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/__init__.py +0 -16
  171. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_aggregate.py +0 -40
  172. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_apply_func.py +0 -41
  173. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_cumulative.py +0 -67
  174. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_describe.py +0 -40
  175. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_groupby.py +0 -55
  176. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_head_tail.py +0 -40
  177. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_index.py +0 -38
  178. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_missing_data.py +0 -55
  179. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_split_apply.py +0 -39
  180. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_stat.py +0 -38
  181. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/__init__.py +0 -16
  182. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_align.py +0 -40
  183. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_base.py +0 -50
  184. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_category.py +0 -73
  185. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_datetime.py +0 -39
  186. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_indexing.py +0 -40
  187. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_reindex.py +0 -40
  188. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_rename.py +0 -40
  189. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_reset_index.py +0 -48
  190. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_timedelta.py +0 -39
  191. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/io/__init__.py +0 -16
  192. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/io/test_parity_io.py +0 -40
  193. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/plot/__init__.py +0 -16
  194. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot.py +0 -45
  195. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_matplotlib.py +0 -45
  196. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_plotly.py +0 -49
  197. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot.py +0 -37
  198. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_matplotlib.py +0 -53
  199. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_plotly.py +0 -45
  200. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/__init__.py +0 -16
  201. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_all_any.py +0 -38
  202. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_arg_ops.py +0 -37
  203. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_as_of.py +0 -37
  204. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_as_type.py +0 -38
  205. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_compute.py +0 -37
  206. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_conversion.py +0 -40
  207. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_cumulative.py +0 -40
  208. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_index.py +0 -38
  209. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_missing_data.py +0 -40
  210. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_series.py +0 -37
  211. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_sort.py +0 -38
  212. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_stat.py +0 -38
  213. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_categorical.py +0 -66
  214. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_config.py +0 -37
  215. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_csv.py +0 -37
  216. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_dataframe_conversion.py +0 -42
  217. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_dataframe_spark_io.py +0 -39
  218. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_default_index.py +0 -49
  219. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_ewm.py +0 -37
  220. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_expanding.py +0 -39
  221. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_extension.py +0 -49
  222. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_frame_spark.py +0 -53
  223. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_generic_functions.py +0 -43
  224. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_indexing.py +0 -49
  225. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_indexops_spark.py +0 -39
  226. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_internal.py +0 -41
  227. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_namespace.py +0 -39
  228. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_numpy_compat.py +0 -60
  229. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_ops_on_diff_frames.py +0 -48
  230. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_ops_on_diff_frames_groupby.py +0 -39
  231. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_ops_on_diff_frames_groupby_expanding.py +0 -44
  232. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_ops_on_diff_frames_groupby_rolling.py +0 -84
  233. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_repr.py +0 -37
  234. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_resample.py +0 -45
  235. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_reshape.py +0 -39
  236. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_rolling.py +0 -39
  237. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_scalars.py +0 -37
  238. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_series_conversion.py +0 -39
  239. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_series_datetime.py +0 -39
  240. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_series_string.py +0 -39
  241. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_spark_functions.py +0 -39
  242. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_sql.py +0 -43
  243. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_stats.py +0 -37
  244. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_typedef.py +0 -36
  245. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_utils.py +0 -37
  246. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_window.py +0 -39
  247. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/__init__.py +0 -16
  248. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_base.py +0 -107
  249. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_binary_ops.py +0 -224
  250. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_boolean_ops.py +0 -825
  251. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_categorical_ops.py +0 -562
  252. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_complex_ops.py +0 -368
  253. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_date_ops.py +0 -257
  254. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_datetime_ops.py +0 -260
  255. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_null_ops.py +0 -178
  256. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_num_arithmetic.py +0 -184
  257. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_num_ops.py +0 -497
  258. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_num_reverse.py +0 -140
  259. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_string_ops.py +0 -354
  260. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_timedelta_ops.py +0 -219
  261. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_udt_ops.py +0 -192
  262. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/testing_utils.py +0 -228
  263. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/__init__.py +0 -16
  264. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_align.py +0 -118
  265. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_basic_slow.py +0 -198
  266. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_cov_corrwith.py +0 -181
  267. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_dot_frame.py +0 -103
  268. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_dot_series.py +0 -141
  269. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_index.py +0 -109
  270. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_series.py +0 -136
  271. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_setitem_frame.py +0 -125
  272. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_setitem_series.py +0 -217
  273. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/__init__.py +0 -16
  274. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_attrs.py +0 -384
  275. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_constructor.py +0 -598
  276. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_conversion.py +0 -73
  277. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_reindexing.py +0 -869
  278. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_reshaping.py +0 -487
  279. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_spark.py +0 -309
  280. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_take.py +0 -156
  281. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_time_series.py +0 -149
  282. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_truncate.py +0 -163
  283. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/__init__.py +0 -16
  284. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_aggregate.py +0 -311
  285. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_apply_func.py +0 -524
  286. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_cumulative.py +0 -419
  287. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_describe.py +0 -144
  288. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_groupby.py +0 -979
  289. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_head_tail.py +0 -234
  290. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_index.py +0 -206
  291. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_missing_data.py +0 -421
  292. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_split_apply.py +0 -187
  293. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_stat.py +0 -397
  294. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/__init__.py +0 -16
  295. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_align.py +0 -100
  296. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_base.py +0 -2743
  297. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_category.py +0 -484
  298. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_datetime.py +0 -276
  299. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_indexing.py +0 -432
  300. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_reindex.py +0 -310
  301. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_rename.py +0 -257
  302. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_reset_index.py +0 -160
  303. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_timedelta.py +0 -128
  304. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/io/__init__.py +0 -16
  305. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/io/test_io.py +0 -137
  306. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/plot/__init__.py +0 -16
  307. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/plot/test_frame_plot.py +0 -170
  308. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/plot/test_frame_plot_matplotlib.py +0 -547
  309. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/plot/test_frame_plot_plotly.py +0 -285
  310. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/plot/test_series_plot.py +0 -106
  311. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/plot/test_series_plot_matplotlib.py +0 -409
  312. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/plot/test_series_plot_plotly.py +0 -247
  313. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/__init__.py +0 -16
  314. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_all_any.py +0 -105
  315. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_arg_ops.py +0 -197
  316. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_as_of.py +0 -137
  317. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_as_type.py +0 -227
  318. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_compute.py +0 -634
  319. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_conversion.py +0 -88
  320. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_cumulative.py +0 -139
  321. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_index.py +0 -475
  322. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_missing_data.py +0 -265
  323. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_series.py +0 -818
  324. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_sort.py +0 -162
  325. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_stat.py +0 -780
  326. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_categorical.py +0 -741
  327. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_config.py +0 -160
  328. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_csv.py +0 -453
  329. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_dataframe_conversion.py +0 -281
  330. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_dataframe_spark_io.py +0 -487
  331. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_default_index.py +0 -109
  332. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_ewm.py +0 -434
  333. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_expanding.py +0 -253
  334. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_extension.py +0 -152
  335. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_frame_spark.py +0 -162
  336. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_generic_functions.py +0 -234
  337. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_indexing.py +0 -1339
  338. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_indexops_spark.py +0 -82
  339. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_internal.py +0 -124
  340. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_namespace.py +0 -638
  341. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_numpy_compat.py +0 -200
  342. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_ops_on_diff_frames.py +0 -1355
  343. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby.py +0 -655
  344. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_expanding.py +0 -113
  345. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_rolling.py +0 -118
  346. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_repr.py +0 -192
  347. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_resample.py +0 -346
  348. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_reshape.py +0 -495
  349. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_rolling.py +0 -263
  350. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_scalars.py +0 -59
  351. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_series_conversion.py +0 -85
  352. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_series_datetime.py +0 -364
  353. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_series_string.py +0 -362
  354. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_spark_functions.py +0 -46
  355. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_sql.py +0 -123
  356. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_stats.py +0 -581
  357. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_typedef.py +0 -447
  358. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_utils.py +0 -301
  359. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_window.py +0 -465
  360. snowflake/snowpark_connect/includes/python/pyspark/resource/tests/__init__.py +0 -16
  361. snowflake/snowpark_connect/includes/python/pyspark/resource/tests/test_resources.py +0 -83
  362. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/__init__.py +0 -16
  363. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/__init__.py +0 -16
  364. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/client/__init__.py +0 -16
  365. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/client/test_artifact.py +0 -420
  366. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/client/test_client.py +0 -358
  367. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/streaming/__init__.py +0 -16
  368. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/streaming/test_parity_foreach.py +0 -36
  369. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/streaming/test_parity_foreach_batch.py +0 -44
  370. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/streaming/test_parity_listener.py +0 -116
  371. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/streaming/test_parity_streaming.py +0 -35
  372. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_connect_basic.py +0 -3612
  373. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_connect_column.py +0 -1042
  374. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_connect_function.py +0 -2381
  375. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_connect_plan.py +0 -1060
  376. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_arrow.py +0 -163
  377. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_arrow_map.py +0 -38
  378. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_arrow_python_udf.py +0 -48
  379. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_catalog.py +0 -36
  380. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_column.py +0 -55
  381. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_conf.py +0 -36
  382. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_dataframe.py +0 -96
  383. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_datasources.py +0 -44
  384. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_errors.py +0 -36
  385. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_functions.py +0 -59
  386. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_group.py +0 -36
  387. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_pandas_cogrouped_map.py +0 -59
  388. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_pandas_grouped_map.py +0 -74
  389. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_pandas_grouped_map_with_state.py +0 -62
  390. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_pandas_map.py +0 -58
  391. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_pandas_udf.py +0 -70
  392. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_pandas_udf_grouped_agg.py +0 -50
  393. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_pandas_udf_scalar.py +0 -68
  394. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_pandas_udf_window.py +0 -40
  395. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_readwriter.py +0 -46
  396. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_serde.py +0 -44
  397. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_types.py +0 -100
  398. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_udf.py +0 -100
  399. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_udtf.py +0 -163
  400. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_session.py +0 -181
  401. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_utils.py +0 -42
  402. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/__init__.py +0 -16
  403. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py +0 -623
  404. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_grouped_map.py +0 -869
  405. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_grouped_map_with_state.py +0 -342
  406. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_map.py +0 -436
  407. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_udf.py +0 -363
  408. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_udf_grouped_agg.py +0 -592
  409. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_udf_scalar.py +0 -1503
  410. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_udf_typehints.py +0 -392
  411. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_udf_typehints_with_future_annotations.py +0 -375
  412. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_udf_window.py +0 -411
  413. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/streaming/__init__.py +0 -16
  414. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/streaming/test_streaming.py +0 -401
  415. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/streaming/test_streaming_foreach.py +0 -295
  416. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/streaming/test_streaming_foreach_batch.py +0 -106
  417. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/streaming/test_streaming_listener.py +0 -558
  418. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_arrow.py +0 -1346
  419. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_arrow_map.py +0 -182
  420. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_arrow_python_udf.py +0 -202
  421. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_catalog.py +0 -503
  422. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_column.py +0 -225
  423. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_conf.py +0 -83
  424. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_context.py +0 -201
  425. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_dataframe.py +0 -1931
  426. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_datasources.py +0 -256
  427. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_errors.py +0 -69
  428. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_functions.py +0 -1349
  429. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_group.py +0 -53
  430. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_pandas_sqlmetrics.py +0 -68
  431. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_readwriter.py +0 -283
  432. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_serde.py +0 -155
  433. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_session.py +0 -412
  434. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_types.py +0 -1581
  435. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_udf.py +0 -961
  436. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_udf_profiler.py +0 -165
  437. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_udtf.py +0 -1456
  438. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_utils.py +0 -1686
  439. snowflake/snowpark_connect/includes/python/pyspark/streaming/tests/__init__.py +0 -16
  440. snowflake/snowpark_connect/includes/python/pyspark/streaming/tests/test_context.py +0 -184
  441. snowflake/snowpark_connect/includes/python/pyspark/streaming/tests/test_dstream.py +0 -706
  442. snowflake/snowpark_connect/includes/python/pyspark/streaming/tests/test_kinesis.py +0 -118
  443. snowflake/snowpark_connect/includes/python/pyspark/streaming/tests/test_listener.py +0 -160
  444. snowflake/snowpark_connect/includes/python/pyspark/tests/__init__.py +0 -16
  445. snowflake/snowpark_connect/includes/python/pyspark/tests/test_appsubmit.py +0 -306
  446. snowflake/snowpark_connect/includes/python/pyspark/tests/test_broadcast.py +0 -196
  447. snowflake/snowpark_connect/includes/python/pyspark/tests/test_conf.py +0 -44
  448. snowflake/snowpark_connect/includes/python/pyspark/tests/test_context.py +0 -346
  449. snowflake/snowpark_connect/includes/python/pyspark/tests/test_daemon.py +0 -89
  450. snowflake/snowpark_connect/includes/python/pyspark/tests/test_install_spark.py +0 -124
  451. snowflake/snowpark_connect/includes/python/pyspark/tests/test_join.py +0 -69
  452. snowflake/snowpark_connect/includes/python/pyspark/tests/test_memory_profiler.py +0 -167
  453. snowflake/snowpark_connect/includes/python/pyspark/tests/test_pin_thread.py +0 -194
  454. snowflake/snowpark_connect/includes/python/pyspark/tests/test_profiler.py +0 -168
  455. snowflake/snowpark_connect/includes/python/pyspark/tests/test_rdd.py +0 -939
  456. snowflake/snowpark_connect/includes/python/pyspark/tests/test_rddbarrier.py +0 -52
  457. snowflake/snowpark_connect/includes/python/pyspark/tests/test_rddsampler.py +0 -66
  458. snowflake/snowpark_connect/includes/python/pyspark/tests/test_readwrite.py +0 -368
  459. snowflake/snowpark_connect/includes/python/pyspark/tests/test_serializers.py +0 -257
  460. snowflake/snowpark_connect/includes/python/pyspark/tests/test_shuffle.py +0 -267
  461. snowflake/snowpark_connect/includes/python/pyspark/tests/test_stage_sched.py +0 -153
  462. snowflake/snowpark_connect/includes/python/pyspark/tests/test_statcounter.py +0 -130
  463. snowflake/snowpark_connect/includes/python/pyspark/tests/test_taskcontext.py +0 -350
  464. snowflake/snowpark_connect/includes/python/pyspark/tests/test_util.py +0 -97
  465. snowflake/snowpark_connect/includes/python/pyspark/tests/test_worker.py +0 -271
  466. snowpark_connect-0.24.0.dist-info/RECORD +0 -898
  467. {snowpark_connect-0.24.0.data → snowpark_connect-0.25.0.data}/scripts/snowpark-connect +0 -0
  468. {snowpark_connect-0.24.0.data → snowpark_connect-0.25.0.data}/scripts/snowpark-session +0 -0
  469. {snowpark_connect-0.24.0.data → snowpark_connect-0.25.0.data}/scripts/snowpark-submit +0 -0
  470. {snowpark_connect-0.24.0.dist-info → snowpark_connect-0.25.0.dist-info}/WHEEL +0 -0
  471. {snowpark_connect-0.24.0.dist-info → snowpark_connect-0.25.0.dist-info}/licenses/LICENSE-binary +0 -0
  472. {snowpark_connect-0.24.0.dist-info → snowpark_connect-0.25.0.dist-info}/licenses/LICENSE.txt +0 -0
  473. {snowpark_connect-0.24.0.dist-info → snowpark_connect-0.25.0.dist-info}/licenses/NOTICE-binary +0 -0
  474. {snowpark_connect-0.24.0.dist-info → snowpark_connect-0.25.0.dist-info}/top_level.txt +0 -0
@@ -1,818 +0,0 @@
1
- #
2
- # Licensed to the Apache Software Foundation (ASF) under one or more
3
- # contributor license agreements. See the NOTICE file distributed with
4
- # this work for additional information regarding copyright ownership.
5
- # The ASF licenses this file to You under the Apache License, Version 2.0
6
- # (the "License"); you may not use this file except in compliance with
7
- # the License. You may obtain a copy of the License at
8
- #
9
- # http://www.apache.org/licenses/LICENSE-2.0
10
- #
11
- # Unless required by applicable law or agreed to in writing, software
12
- # distributed under the License is distributed on an "AS IS" BASIS,
13
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
- # See the License for the specific language governing permissions and
15
- # limitations under the License.
16
- #
17
-
18
- import unittest
19
- from collections import defaultdict
20
- from distutils.version import LooseVersion
21
- import inspect
22
-
23
- from datetime import datetime, timedelta
24
-
25
- import numpy as np
26
- import pandas as pd
27
- from pyspark.ml.linalg import SparseVector
28
-
29
- from pyspark import pandas as ps
30
- from pyspark.testing.pandasutils import (
31
- have_tabulate,
32
- PandasOnSparkTestCase,
33
- SPARK_CONF_ARROW_ENABLED,
34
- tabulate_requirement_message,
35
- )
36
- from pyspark.testing.sqlutils import SQLTestUtils
37
- from pyspark.pandas.exceptions import PandasNotImplementedError
38
- from pyspark.pandas.missing.series import MissingPandasLikeSeries
39
- from pyspark.pandas.typedef.typehints import (
40
- extension_dtypes,
41
- extension_dtypes_available,
42
- extension_float_dtypes_available,
43
- extension_object_dtypes_available,
44
- )
45
-
46
-
47
- class SeriesTestsMixin:
48
- @property
49
- def pser(self):
50
- return pd.Series([1, 2, 3, 4, 5, 6, 7], name="x")
51
-
52
- @property
53
- def psser(self):
54
- return ps.from_pandas(self.pser)
55
-
56
- def test_creation_index(self):
57
- err_msg = (
58
- "The given index cannot be a pandas-on-Spark index. Try pandas index or array-like."
59
- )
60
- with self.assertRaisesRegex(TypeError, err_msg):
61
- ps.Series([1, 2], index=ps.Index([1, 2]))
62
-
63
- with self.assertRaisesRegex(TypeError, err_msg):
64
- ps.Series([1, 2], index=ps.MultiIndex.from_tuples([(1, 3), (2, 4)]))
65
-
66
- def test_series_ops(self):
67
- pser = self.pser
68
- psser = self.psser
69
-
70
- self.assert_eq(psser + 1 + 10 * psser, pser + 1 + 10 * pser)
71
- self.assert_eq(psser + 1 + 10 * psser.index, pser + 1 + 10 * pser.index)
72
- self.assert_eq(psser.index + 1 + 10 * psser, pser.index + 1 + 10 * pser)
73
-
74
- def test_series_tuple_name(self):
75
- pser = self.pser
76
- pser.name = ("x", "a")
77
-
78
- psser = ps.from_pandas(pser)
79
-
80
- self.assert_eq(psser, pser)
81
- self.assert_eq(psser.name, pser.name)
82
-
83
- pser.name = ("y", "z")
84
- psser.name = ("y", "z")
85
-
86
- self.assert_eq(psser, pser)
87
- self.assert_eq(psser.name, pser.name)
88
-
89
- def test_repr_cache_invalidation(self):
90
- # If there is any cache, inplace operations should invalidate it.
91
- s = ps.range(10)["id"]
92
- s.__repr__()
93
- s.rename("a", inplace=True)
94
- self.assertEqual(s.__repr__(), s.rename("a").__repr__())
95
-
96
- def _check_extension(self, psser, pser):
97
- if LooseVersion("1.1") <= LooseVersion(pd.__version__) < LooseVersion("1.2.2"):
98
- self.assert_eq(psser, pser, check_exact=False)
99
- self.assertTrue(isinstance(psser.dtype, extension_dtypes))
100
- else:
101
- self.assert_eq(psser, pser)
102
-
103
- def test_empty_series(self):
104
- pser_a = pd.Series([], dtype="i1")
105
- pser_b = pd.Series([], dtype="str")
106
-
107
- self.assert_eq(ps.from_pandas(pser_a), pser_a)
108
-
109
- psser_b = ps.from_pandas(pser_b)
110
- self.assert_eq(psser_b, pser_b)
111
-
112
- with self.sql_conf({SPARK_CONF_ARROW_ENABLED: False}):
113
- self.assert_eq(ps.from_pandas(pser_a), pser_a)
114
- self.assert_eq(ps.from_pandas(pser_b), pser_b)
115
-
116
- def test_all_null_series(self):
117
- pser_a = pd.Series([None, None, None], dtype="float64")
118
- pser_b = pd.Series([None, None, None], dtype="str")
119
-
120
- self.assert_eq(ps.from_pandas(pser_a), pser_a)
121
-
122
- psser_b = ps.from_pandas(pser_b)
123
- self.assert_eq(psser_b, pser_b)
124
-
125
- with self.sql_conf({SPARK_CONF_ARROW_ENABLED: False}):
126
- self.assert_eq(ps.from_pandas(pser_a), pser_a)
127
- self.assert_eq(ps.from_pandas(pser_b), pser_b)
128
-
129
- def test_head(self):
130
- psser = self.psser
131
- pser = self.pser
132
-
133
- self.assert_eq(psser.head(3), pser.head(3))
134
- self.assert_eq(psser.head(0), pser.head(0))
135
- self.assert_eq(psser.head(-3), pser.head(-3))
136
- self.assert_eq(psser.head(-10), pser.head(-10))
137
-
138
- def test_last(self):
139
- with self.assertRaises(TypeError):
140
- self.psser.last("1D")
141
-
142
- index = pd.date_range("2018-04-09", periods=4, freq="2D")
143
- pser = pd.Series([1, 2, 3, 4], index=index)
144
- psser = ps.from_pandas(pser)
145
- self.assert_eq(psser.last("1D"), pser.last("1D"))
146
-
147
- def test_first(self):
148
- with self.assertRaises(TypeError):
149
- self.psser.first("1D")
150
-
151
- index = pd.date_range("2018-04-09", periods=4, freq="2D")
152
- pser = pd.Series([1, 2, 3, 4], index=index)
153
- psser = ps.from_pandas(pser)
154
- self.assert_eq(psser.first("1D"), pser.first("1D"))
155
-
156
- def test_rename(self):
157
- pser = pd.Series([1, 2, 3, 4, 5, 6, 7], name="x")
158
- psser = ps.from_pandas(pser)
159
-
160
- pser.name = "renamed"
161
- psser.name = "renamed"
162
- self.assertEqual(psser.name, "renamed")
163
- self.assert_eq(psser, pser)
164
-
165
- pser.name = None
166
- psser.name = None
167
- self.assertEqual(psser.name, None)
168
- self.assert_eq(psser, pser)
169
-
170
- pidx = pser.index
171
- psidx = psser.index
172
- pidx.name = "renamed"
173
- psidx.name = "renamed"
174
- self.assertEqual(psidx.name, "renamed")
175
- self.assert_eq(psidx, pidx)
176
-
177
- expected_error_message = "Series.name must be a hashable type"
178
- with self.assertRaisesRegex(TypeError, expected_error_message):
179
- psser.name = ["renamed"]
180
- with self.assertRaisesRegex(TypeError, expected_error_message):
181
- psser.name = ["0", "1"]
182
- with self.assertRaisesRegex(TypeError, expected_error_message):
183
- ps.Series([1, 2, 3], name=["0", "1"])
184
-
185
- def test_rename_method(self):
186
- # Series name
187
- pdf = pd.DataFrame({"x": [1, 2, 3, 4, 5, 6, 7]})
188
- psdf = ps.from_pandas(pdf)
189
- pser, psser = pdf.x, psdf.x
190
-
191
- self.assert_eq(psser.rename("y"), pser.rename("y"))
192
- self.assertEqual(psser.name, "x") # no mutation
193
- self.assert_eq(psser.rename(), pser.rename())
194
-
195
- self.assert_eq((psser.rename("y") + 1).head(), (pser.rename("y") + 1).head())
196
-
197
- psser.rename("z", inplace=True)
198
- pser.rename("z", inplace=True)
199
- self.assertEqual(psser.name, "z")
200
- self.assert_eq(psser, pser)
201
- self.assert_eq(psdf, pdf)
202
-
203
- expected_error_message = "Series.name must be a hashable type"
204
- with self.assertRaisesRegex(TypeError, expected_error_message):
205
- psser.rename(["0", "1"])
206
-
207
- # Function index
208
- self.assert_eq(psser.rename(lambda x: x**2), pser.rename(lambda x: x**2))
209
- self.assert_eq((psser + 1).rename(lambda x: x**2), (pser + 1).rename(lambda x: x**2))
210
-
211
- expected_error_message = "inplace True is not supported yet for a function 'index'"
212
- with self.assertRaisesRegex(ValueError, expected_error_message):
213
- psser.rename(lambda x: x**2, inplace=True)
214
-
215
- unsupported_index_inputs = (pd.Series([2, 3, 4, 5, 6, 7, 8]), {0: "zero", 1: "one"})
216
- for index in unsupported_index_inputs:
217
- expected_error_message = (
218
- "'index' of %s type is not supported yet" % type(index).__name__
219
- )
220
- with self.assertRaisesRegex(ValueError, expected_error_message):
221
- psser.rename(index)
222
-
223
- # Series index
224
- # pser = pd.Series(['a', 'b', 'c', 'd', 'e', 'f', 'g'], name='x')
225
- # psser = ps.from_pandas(s)
226
-
227
- # res = psser.rename(pser)
228
- # self.assert_eq(res, pser.rename(pser))
229
-
230
- # res = psser.rename(psser)
231
- # self.assert_eq(res, pser.rename(pser))
232
-
233
- # res = psser.rename(lambda x: x**2, inplace=True)
234
- # self.assertis(res, psser)
235
- # s.rename(lambda x: x**2, inplace=True)
236
- # self.assert_eq(psser, pser)
237
-
238
- def test_or(self):
239
- pdf = pd.DataFrame(
240
- {
241
- "left": [True, False, True, False, np.nan, np.nan, True, False, np.nan],
242
- "right": [True, False, False, True, True, False, np.nan, np.nan, np.nan],
243
- }
244
- )
245
- psdf = ps.from_pandas(pdf)
246
-
247
- self.assert_eq(psdf["left"] | psdf["right"], pdf["left"] | pdf["right"])
248
- self.assert_eq(psdf["left"] | True, pdf["left"] | True)
249
- self.assert_eq(psdf["left"] | False, pdf["left"] | False)
250
- self.assert_eq(psdf["left"] | None, pdf["left"] | None)
251
- self.assert_eq(True | psdf["right"], True | pdf["right"])
252
- self.assert_eq(False | psdf["right"], False | pdf["right"])
253
- self.assert_eq(None | psdf["right"], None | pdf["right"])
254
-
255
- @unittest.skipIf(
256
- not extension_object_dtypes_available, "pandas extension object dtypes are not available"
257
- )
258
- def test_or_extenstion_dtypes(self):
259
- pdf = pd.DataFrame(
260
- {
261
- "left": [True, False, True, False, np.nan, np.nan, True, False, np.nan],
262
- "right": [True, False, False, True, True, False, np.nan, np.nan, np.nan],
263
- }
264
- ).astype("boolean")
265
- psdf = ps.from_pandas(pdf)
266
-
267
- self._check_extension(psdf["left"] | psdf["right"], pdf["left"] | pdf["right"])
268
- self._check_extension(psdf["left"] | True, pdf["left"] | True)
269
- self._check_extension(psdf["left"] | False, pdf["left"] | False)
270
- self._check_extension(psdf["left"] | pd.NA, pdf["left"] | pd.NA)
271
- self._check_extension(True | psdf["right"], True | pdf["right"])
272
- self._check_extension(False | psdf["right"], False | pdf["right"])
273
- self._check_extension(pd.NA | psdf["right"], pd.NA | pdf["right"])
274
-
275
- def test_and(self):
276
- pdf = pd.DataFrame(
277
- {
278
- "left": [True, False, True, False, np.nan, np.nan, True, False, np.nan],
279
- "right": [True, False, False, True, True, False, np.nan, np.nan, np.nan],
280
- }
281
- )
282
- psdf = ps.from_pandas(pdf)
283
-
284
- self.assert_eq(psdf["left"] & psdf["right"], pdf["left"] & pdf["right"])
285
- self.assert_eq(psdf["left"] & True, pdf["left"] & True)
286
- self.assert_eq(psdf["left"] & False, pdf["left"] & False)
287
- self.assert_eq(psdf["left"] & None, pdf["left"] & None)
288
- self.assert_eq(True & psdf["right"], True & pdf["right"])
289
- self.assert_eq(False & psdf["right"], False & pdf["right"])
290
- self.assert_eq(None & psdf["right"], None & pdf["right"])
291
-
292
- @unittest.skipIf(
293
- not extension_object_dtypes_available, "pandas extension object dtypes are not available"
294
- )
295
- def test_and_extenstion_dtypes(self):
296
- pdf = pd.DataFrame(
297
- {
298
- "left": [True, False, True, False, np.nan, np.nan, True, False, np.nan],
299
- "right": [True, False, False, True, True, False, np.nan, np.nan, np.nan],
300
- }
301
- ).astype("boolean")
302
- psdf = ps.from_pandas(pdf)
303
-
304
- self._check_extension(psdf["left"] & psdf["right"], pdf["left"] & pdf["right"])
305
- self._check_extension(psdf["left"] & True, pdf["left"] & True)
306
- self._check_extension(psdf["left"] & False, pdf["left"] & False)
307
- self._check_extension(psdf["left"] & pd.NA, pdf["left"] & pd.NA)
308
- self._check_extension(True & psdf["right"], True & pdf["right"])
309
- self._check_extension(False & psdf["right"], False & pdf["right"])
310
- self._check_extension(pd.NA & psdf["right"], pd.NA & pdf["right"])
311
-
312
- def test_isin(self):
313
- pser = pd.Series(["lama", "cow", "lama", "beetle", "lama", "hippo"], name="animal")
314
-
315
- psser = ps.from_pandas(pser)
316
-
317
- self.assert_eq(psser.isin(["cow", "lama"]), pser.isin(["cow", "lama"]))
318
- self.assert_eq(psser.isin(np.array(["cow", "lama"])), pser.isin(np.array(["cow", "lama"])))
319
- self.assert_eq(psser.isin({"cow"}), pser.isin({"cow"}))
320
-
321
- pser = pd.Series([np.int64(1), np.int32(1), 1])
322
- psser = ps.from_pandas(pser)
323
- self.assert_eq(psser.isin([np.int64(1)]), pser.isin([np.int64(1)]))
324
-
325
- msg = "only list-like objects are allowed to be passed to isin()"
326
- with self.assertRaisesRegex(TypeError, msg):
327
- psser.isin(1)
328
-
329
- # when Series have NaN
330
- pser = pd.Series(["lama", "cow", None, "lama", "beetle", "lama", "hippo", None], name="a")
331
- psser = ps.from_pandas(pser)
332
-
333
- self.assert_eq(psser.isin(["cow", "lama"]), pser.isin(["cow", "lama"]))
334
-
335
- pser = pd.Series([None, 5, None, 3, 2, 1, None, 0, 0], name="a")
336
- psser = ps.from_pandas(pser)
337
-
338
- if LooseVersion(pd.__version__) >= LooseVersion("1.2"):
339
- self.assert_eq(psser.isin([1, 5, 0, None]), pser.isin([1, 5, 0, None]))
340
- else:
341
- expected = pd.Series(
342
- [False, True, False, False, False, True, False, True, True], name="a"
343
- )
344
- self.assert_eq(psser.isin([1, 5, 0, None]), expected)
345
-
346
- def test_notnull(self):
347
- pser = pd.Series([1, 2, 3, 4, np.nan, 6], name="x")
348
- psser = ps.from_pandas(pser)
349
-
350
- self.assert_eq(psser.notnull(), pser.notnull())
351
-
352
- pser = self.pser
353
- psser = self.psser
354
-
355
- self.assert_eq(psser.notnull(), pser.notnull())
356
-
357
- def test_missing(self):
358
- psser = self.psser
359
-
360
- missing_functions = inspect.getmembers(MissingPandasLikeSeries, inspect.isfunction)
361
- unsupported_functions = [
362
- name for (name, type_) in missing_functions if type_.__name__ == "unsupported_function"
363
- ]
364
- for name in unsupported_functions:
365
- with self.assertRaisesRegex(
366
- PandasNotImplementedError,
367
- "method.*Series.*{}.*not implemented( yet\\.|\\. .+)".format(name),
368
- ):
369
- getattr(psser, name)()
370
-
371
- deprecated_functions = [
372
- name for (name, type_) in missing_functions if type_.__name__ == "deprecated_function"
373
- ]
374
- for name in deprecated_functions:
375
- with self.assertRaisesRegex(
376
- PandasNotImplementedError, "method.*Series.*{}.*is deprecated".format(name)
377
- ):
378
- getattr(psser, name)()
379
-
380
- missing_properties = inspect.getmembers(
381
- MissingPandasLikeSeries, lambda o: isinstance(o, property)
382
- )
383
- unsupported_properties = [
384
- name
385
- for (name, type_) in missing_properties
386
- if type_.fget.__name__ == "unsupported_property"
387
- ]
388
- for name in unsupported_properties:
389
- with self.assertRaisesRegex(
390
- PandasNotImplementedError,
391
- "property.*Series.*{}.*not implemented( yet\\.|\\. .+)".format(name),
392
- ):
393
- getattr(psser, name)
394
- deprecated_properties = [
395
- name
396
- for (name, type_) in missing_properties
397
- if type_.fget.__name__ == "deprecated_property"
398
- ]
399
- for name in deprecated_properties:
400
- with self.assertRaisesRegex(
401
- PandasNotImplementedError, "property.*Series.*{}.*is deprecated".format(name)
402
- ):
403
- getattr(psser, name)
404
-
405
- def test_map(self):
406
- pser = pd.Series(["cat", "dog", None, "rabbit"])
407
- psser = ps.from_pandas(pser)
408
-
409
- # dict correspondence
410
- # Currently pandas API on Spark doesn't return NaN as pandas does.
411
- self.assert_eq(psser.map({}), pser.map({}).replace({np.nan: None}))
412
-
413
- d = defaultdict(lambda: "abc")
414
- self.assertTrue("abc" in repr(psser.map(d)))
415
- self.assert_eq(psser.map(d), pser.map(d))
416
-
417
- # series correspondence
418
- pser_to_apply = pd.Series(["one", "two", "four"], index=["cat", "dog", "rabbit"])
419
- self.assert_eq(psser.map(pser_to_apply), pser.map(pser_to_apply))
420
- self.assert_eq(
421
- psser.map(pser_to_apply, na_action="ignore"),
422
- pser.map(pser_to_apply, na_action="ignore"),
423
- )
424
-
425
- # function correspondence
426
- self.assert_eq(
427
- psser.map(lambda x: x.upper(), na_action="ignore"),
428
- pser.map(lambda x: x.upper(), na_action="ignore"),
429
- )
430
-
431
- def to_upper(string) -> str:
432
- return string.upper() if string else ""
433
-
434
- self.assert_eq(psser.map(to_upper), pser.map(to_upper))
435
-
436
- def tomorrow(date) -> datetime:
437
- return date + timedelta(days=1)
438
-
439
- pser = pd.Series([datetime(2019, 10, 24)])
440
- psser = ps.from_pandas(pser)
441
- self.assert_eq(psser.map(tomorrow), pser.map(tomorrow))
442
-
443
- def test_add_prefix(self):
444
- pser = pd.Series([1, 2, 3, 4], name="0")
445
- psser = ps.from_pandas(pser)
446
- self.assert_eq(pser.add_prefix("item_"), psser.add_prefix("item_"))
447
-
448
- pser = pd.Series(
449
- [1, 2, 3],
450
- name="0",
451
- index=pd.MultiIndex.from_tuples([("A", "X"), ("A", "Y"), ("B", "X")]),
452
- )
453
- psser = ps.from_pandas(pser)
454
- self.assert_eq(pser.add_prefix("item_"), psser.add_prefix("item_"))
455
-
456
- def test_add_suffix(self):
457
- pser = pd.Series([1, 2, 3, 4], name="0")
458
- psser = ps.from_pandas(pser)
459
- self.assert_eq(pser.add_suffix("_item"), psser.add_suffix("_item"))
460
-
461
- pser = pd.Series(
462
- [1, 2, 3],
463
- name="0",
464
- index=pd.MultiIndex.from_tuples([("A", "X"), ("A", "Y"), ("B", "X")]),
465
- )
466
- psser = ps.from_pandas(pser)
467
- self.assert_eq(pser.add_suffix("_item"), psser.add_suffix("_item"))
468
-
469
- def test_xs(self):
470
- midx = pd.MultiIndex(
471
- [["a", "b", "c"], ["lama", "cow", "falcon"], ["speed", "weight", "length"]],
472
- [[0, 0, 0, 1, 1, 1, 2, 2, 2], [0, 0, 0, 1, 1, 1, 2, 2, 2], [0, 1, 2, 0, 1, 2, 0, 1, 2]],
473
- )
474
- pser = pd.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3], index=midx)
475
- psser = ps.from_pandas(pser)
476
-
477
- self.assert_eq(psser.xs(("a", "lama", "speed")), pser.xs(("a", "lama", "speed")))
478
-
479
- def test_update(self):
480
- pser = pd.Series([10, 20, 15, 30, 45], name="x")
481
- psser = ps.Series(pser)
482
-
483
- msg = "'other' must be a Series"
484
- with self.assertRaisesRegex(TypeError, msg):
485
- psser.update(10)
486
-
487
- def _get_data():
488
- pdf = pd.DataFrame(
489
- {
490
- "a": [None, 2, 3, 4, 5, 6, 7, 8, None],
491
- "b": [None, 5, None, 3, 2, 1, None, 0, 0],
492
- "c": [1, 5, 1, 3, 2, 1, 1, 0, 0],
493
- },
494
- )
495
- psdf = ps.from_pandas(pdf)
496
- return pdf, psdf
497
-
498
- pdf, psdf = _get_data()
499
-
500
- psdf.a.update(psdf.a)
501
- pdf.a.update(pdf.a)
502
- self.assert_eq(psdf, pdf)
503
-
504
- pdf, psdf = _get_data()
505
-
506
- psdf.a.update(psdf.b)
507
- pdf.a.update(pdf.b)
508
- self.assert_eq(psdf, pdf)
509
-
510
- pdf, psdf = _get_data()
511
- pser = pdf.a
512
- psser = psdf.a
513
-
514
- pser.update(pdf.b)
515
- psser.update(psdf.b)
516
- self.assert_eq(psser, pser)
517
- self.assert_eq(psdf, pdf)
518
-
519
- def test_where(self):
520
- pser1 = pd.Series([0, 1, 2, 3, 4])
521
- psser1 = ps.from_pandas(pser1)
522
-
523
- self.assert_eq(pser1.where(pser1 > 3), psser1.where(psser1 > 3).sort_index())
524
-
525
- def test_mask(self):
526
- pser1 = pd.Series([0, 1, 2, 3, 4])
527
- psser1 = ps.from_pandas(pser1)
528
-
529
- self.assert_eq(pser1.mask(pser1 > 3), psser1.mask(psser1 > 3).sort_index())
530
-
531
- def test_getitem(self):
532
- pser = pd.Series([10, 20, 15, 30, 45], ["A", "A", "B", "C", "D"])
533
- psser = ps.Series(pser)
534
-
535
- self.assert_eq(psser["A"], pser["A"])
536
- self.assert_eq(psser["B"], pser["B"])
537
- self.assert_eq(psser[psser > 15], pser[pser > 15])
538
-
539
- # for MultiIndex
540
- midx = pd.MultiIndex(
541
- [["a", "b", "c"], ["lama", "cow", "falcon"], ["speed", "weight", "length"]],
542
- [[0, 0, 0, 0, 0, 0, 1, 1, 1], [0, 0, 0, 1, 1, 1, 2, 2, 2], [0, 0, 0, 0, 1, 2, 0, 1, 2]],
543
- )
544
- pser = pd.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3], name="0", index=midx)
545
- psser = ps.Series(pser)
546
-
547
- self.assert_eq(psser["a"], pser["a"])
548
- self.assert_eq(psser["a", "lama"], pser["a", "lama"])
549
- self.assert_eq(psser[psser > 1.5], pser[pser > 1.5])
550
-
551
- msg = r"'Key length \(4\) exceeds index depth \(3\)'"
552
- with self.assertRaisesRegex(KeyError, msg):
553
- psser[("a", "lama", "speed", "x")]
554
-
555
- def test_keys(self):
556
- midx = pd.MultiIndex(
557
- [["lama", "cow", "falcon"], ["speed", "weight", "length"]],
558
- [[0, 0, 0, 1, 1, 1, 2, 2, 2], [0, 1, 2, 0, 1, 2, 0, 1, 2]],
559
- )
560
- pser = pd.Series([45, 200, 1.2, 30, 250, 1.5, 320, 1, 0.3], index=midx)
561
- psser = ps.from_pandas(pser)
562
-
563
- self.assert_eq(psser.keys(), pser.keys())
564
-
565
- def test_udt(self):
566
- sparse_values = {0: 0.1, 1: 1.1}
567
- sparse_vector = SparseVector(len(sparse_values), sparse_values)
568
- pser = pd.Series([sparse_vector])
569
- psser = ps.from_pandas(pser)
570
- self.assert_eq(psser, pser)
571
-
572
- def test_repeat(self):
573
- pser = pd.Series(["a", "b", "c"], name="0", index=np.random.rand(3))
574
- psser = ps.from_pandas(pser)
575
-
576
- self.assert_eq(psser.repeat(3).sort_index(), pser.repeat(3).sort_index())
577
- self.assert_eq(psser.repeat(0).sort_index(), pser.repeat(0).sort_index())
578
-
579
- self.assertRaises(ValueError, lambda: psser.repeat(-1))
580
- self.assertRaises(TypeError, lambda: psser.repeat("abc"))
581
-
582
- pdf = pd.DataFrame({"a": ["a", "b", "c"], "rep": [10, 20, 30]}, index=np.random.rand(3))
583
- psdf = ps.from_pandas(pdf)
584
-
585
- self.assert_eq(psdf.a.repeat(psdf.rep).sort_index(), pdf.a.repeat(pdf.rep).sort_index())
586
-
587
- def test_take(self):
588
- pser = pd.Series([100, 200, 300, 400, 500], name="Koalas")
589
- psser = ps.from_pandas(pser)
590
-
591
- self.assert_eq(psser.take([0, 2, 4]).sort_values(), pser.take([0, 2, 4]).sort_values())
592
- self.assert_eq(
593
- psser.take(range(0, 5, 2)).sort_values(), pser.take(range(0, 5, 2)).sort_values()
594
- )
595
- self.assert_eq(psser.take([-4, -2, 0]).sort_values(), pser.take([-4, -2, 0]).sort_values())
596
- self.assert_eq(
597
- psser.take(range(-2, 1, 2)).sort_values(), pser.take(range(-2, 1, 2)).sort_values()
598
- )
599
-
600
- # Checking the type of indices.
601
- self.assertRaises(TypeError, lambda: psser.take(1))
602
- self.assertRaises(TypeError, lambda: psser.take("1"))
603
- self.assertRaises(TypeError, lambda: psser.take({1, 2}))
604
- self.assertRaises(TypeError, lambda: psser.take({1: None, 2: None}))
605
-
606
- def test_squeeze(self):
607
- # Single value
608
- pser = pd.Series([90])
609
- psser = ps.from_pandas(pser)
610
- self.assert_eq(psser.squeeze(), pser.squeeze())
611
-
612
- # Single value with MultiIndex
613
- midx = pd.MultiIndex.from_tuples([("a", "b", "c")])
614
- pser = pd.Series([90], index=midx)
615
- psser = ps.from_pandas(pser)
616
- self.assert_eq(psser.squeeze(), pser.squeeze())
617
-
618
- # Multiple values
619
- pser = pd.Series([90, 91, 85])
620
- psser = ps.from_pandas(pser)
621
- self.assert_eq(psser.squeeze(), pser.squeeze())
622
-
623
- # Multiple values with MultiIndex
624
- midx = pd.MultiIndex.from_tuples([("a", "x"), ("b", "y"), ("c", "z")])
625
- pser = pd.Series([90, 91, 85], index=midx)
626
- psser = ps.from_pandas(pser)
627
- self.assert_eq(psser.squeeze(), pser.squeeze())
628
-
629
- def test_shape(self):
630
- pser = pd.Series(["a", "b", "c"])
631
- psser = ps.from_pandas(pser)
632
-
633
- self.assert_eq(pser.shape, psser.shape)
634
-
635
- # for MultiIndex
636
- midx = pd.MultiIndex.from_tuples([("a", "x"), ("b", "y"), ("c", "z")])
637
- pser = pd.Series(["a", "b", "c"], index=midx)
638
- psser = ps.from_pandas(pser)
639
-
640
- self.assert_eq(pser.shape, psser.shape)
641
-
642
- def test_item(self):
643
- psser = ps.Series([10, 20])
644
- self.assertRaises(ValueError, lambda: psser.item())
645
-
646
- def test_filter(self):
647
- pser = pd.Series([0, 1, 2], index=["one", "two", "three"])
648
- psser = ps.from_pandas(pser)
649
-
650
- self.assert_eq(pser.filter(items=["one", "three"]), psser.filter(items=["one", "three"]))
651
- self.assert_eq(pser.filter(regex="e$"), psser.filter(regex="e$"))
652
- self.assert_eq(pser.filter(like="hre"), psser.filter(like="hre"))
653
-
654
- with self.assertRaisesRegex(ValueError, "Series does not support columns axis."):
655
- psser.filter(like="hre", axis=1)
656
-
657
- # for MultiIndex
658
- midx = pd.MultiIndex.from_tuples([("one", "x"), ("two", "y"), ("three", "z")])
659
- pser = pd.Series([0, 1, 2], index=midx)
660
- psser = ps.from_pandas(pser)
661
-
662
- self.assert_eq(
663
- pser.filter(items=[("one", "x"), ("three", "z")]),
664
- psser.filter(items=[("one", "x"), ("three", "z")]),
665
- )
666
-
667
- with self.assertRaisesRegex(TypeError, "Unsupported type list"):
668
- psser.filter(items=[["one", "x"], ("three", "z")])
669
-
670
- with self.assertRaisesRegex(ValueError, "The item should not be empty."):
671
- psser.filter(items=[(), ("three", "z")])
672
-
673
- @unittest.skipIf(
674
- LooseVersion(pd.__version__) >= LooseVersion("2.0.0"),
675
- "TODO(SPARK-43480): Enable SeriesTests.test_iteritems for pandas 2.0.0.",
676
- )
677
- def test_iteritems(self):
678
- pser = pd.Series(["A", "B", "C"])
679
- psser = ps.from_pandas(pser)
680
-
681
- for (p_name, p_items), (k_name, k_items) in zip(pser.iteritems(), psser.iteritems()):
682
- self.assert_eq(p_name, k_name)
683
- self.assert_eq(p_items, k_items)
684
-
685
- def test_dot(self):
686
- pdf = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
687
- psdf = ps.from_pandas(pdf)
688
-
689
- self.assert_eq((psdf["b"] * 10).dot(psdf["a"]), (pdf["b"] * 10).dot(pdf["a"]))
690
- self.assert_eq((psdf["b"] * 10).dot(psdf), (pdf["b"] * 10).dot(pdf))
691
- self.assert_eq((psdf["b"] * 10).dot(psdf + 1), (pdf["b"] * 10).dot(pdf + 1))
692
-
693
- psdf_other = ps.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}, index=["x", "y", "z"])
694
- with self.assertRaisesRegex(ValueError, "matrices are not aligned"):
695
- psdf["b"].dot(psdf_other)
696
-
697
- def test_tail(self):
698
- pser = pd.Series(range(1000), name="Koalas")
699
- psser = ps.from_pandas(pser)
700
-
701
- self.assert_eq(pser.tail(), psser.tail())
702
- self.assert_eq(pser.tail(10), psser.tail(10))
703
- self.assert_eq(pser.tail(-990), psser.tail(-990))
704
- self.assert_eq(pser.tail(0), psser.tail(0))
705
- self.assert_eq(pser.tail(1001), psser.tail(1001))
706
- self.assert_eq(pser.tail(-1001), psser.tail(-1001))
707
- self.assert_eq((pser + 1).tail(), (psser + 1).tail())
708
- self.assert_eq((pser + 1).tail(10), (psser + 1).tail(10))
709
- self.assert_eq((pser + 1).tail(-990), (psser + 1).tail(-990))
710
- self.assert_eq((pser + 1).tail(0), (psser + 1).tail(0))
711
- self.assert_eq((pser + 1).tail(1001), (psser + 1).tail(1001))
712
- self.assert_eq((pser + 1).tail(-1001), (psser + 1).tail(-1001))
713
- with self.assertRaisesRegex(TypeError, "bad operand type for unary -: 'str'"):
714
- psser.tail("10")
715
-
716
- def test_apply(self):
717
- psser = self.psser
718
-
719
- def udf(col) -> ps.Series[int]:
720
- return col + 10
721
-
722
- with self.assertRaisesRegex(
723
- ValueError,
724
- r"Expected the return type of this function to be of scalar type, "
725
- r"but found type SeriesType\[LongType\(\)\]",
726
- ):
727
- psser.apply(udf)
728
-
729
- def test_combine_first(self):
730
- pdf = pd.DataFrame(
731
- {
732
- "A": {"falcon": 330.0, "eagle": 160.0},
733
- "B": {"falcon": 345.0, "eagle": 200.0, "duck": 30.0},
734
- }
735
- )
736
- pser1, pser2 = pdf.A, pdf.B
737
- psdf = ps.from_pandas(pdf)
738
- psser1, psser2 = psdf.A, psdf.B
739
-
740
- self.assert_eq(psser1.combine_first(psser2), pser1.combine_first(pser2))
741
-
742
- psser1.name = pser1.name = ("X", "A")
743
- psser2.name = pser2.name = ("Y", "B")
744
-
745
- self.assert_eq(psser1.combine_first(psser2), pser1.combine_first(pser2))
746
-
747
- def test_eq(self):
748
- pser = pd.Series([1, 2, 3, 4, 5, 6], name="x")
749
- psser = ps.from_pandas(pser)
750
-
751
- # other = Series
752
- self.assert_eq(pser.eq(pser), psser.eq(psser))
753
- self.assert_eq(pser == pser, psser == psser)
754
-
755
- # other = dict
756
- other = {1: None, 2: None, 3: None, 4: None, np.nan: None, 6: None}
757
- self.assert_eq(pser.eq(other), psser.eq(other))
758
- self.assert_eq(pser == other, psser == other)
759
-
760
- # other = set
761
- other = {1, 2, 3, 4, np.nan, 6}
762
- self.assert_eq(pser.eq(other), psser.eq(other))
763
- self.assert_eq(pser == other, psser == other)
764
-
765
- # other = list
766
- other = [np.nan, 1, 3, 4, np.nan, 6]
767
- if LooseVersion(pd.__version__) >= LooseVersion("1.2"):
768
- self.assert_eq(pser.eq(other), psser.eq(other).sort_index())
769
- self.assert_eq(pser == other, (psser == other).sort_index())
770
- else:
771
- self.assert_eq(pser.eq(other).rename("x"), psser.eq(other).sort_index())
772
- self.assert_eq((pser == other).rename("x"), (psser == other).sort_index())
773
-
774
- # other = tuple
775
- other = (np.nan, 1, 3, 4, np.nan, 6)
776
- if LooseVersion(pd.__version__) >= LooseVersion("1.2"):
777
- self.assert_eq(pser.eq(other), psser.eq(other).sort_index())
778
- self.assert_eq(pser == other, (psser == other).sort_index())
779
- else:
780
- self.assert_eq(pser.eq(other).rename("x"), psser.eq(other).sort_index())
781
- self.assert_eq((pser == other).rename("x"), (psser == other).sort_index())
782
-
783
- # other = list with the different length
784
- other = [np.nan, 1, 3, 4, np.nan]
785
- with self.assertRaisesRegex(ValueError, "Lengths must be equal"):
786
- psser.eq(other)
787
- with self.assertRaisesRegex(ValueError, "Lengths must be equal"):
788
- psser == other
789
-
790
- # other = tuple with the different length
791
- other = (np.nan, 1, 3, 4, np.nan)
792
- with self.assertRaisesRegex(ValueError, "Lengths must be equal"):
793
- psser.eq(other)
794
- with self.assertRaisesRegex(ValueError, "Lengths must be equal"):
795
- psser == other
796
-
797
- def test_transform(self):
798
- psser = self.psser
799
- with self.assertRaisesRegex(
800
- NotImplementedError, 'axis should be either 0 or "index" currently.'
801
- ):
802
- psser.transform(lambda x: x + 1, axis=1)
803
-
804
-
805
- class SeriesTests(SeriesTestsMixin, PandasOnSparkTestCase, SQLTestUtils):
806
- pass
807
-
808
-
809
- if __name__ == "__main__":
810
- from pyspark.pandas.tests.series.test_series import * # noqa: F401
811
-
812
- try:
813
- import xmlrunner
814
-
815
- testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
816
- except ImportError:
817
- testRunner = None
818
- unittest.main(testRunner=testRunner, verbosity=2)