snowpark-connect 0.24.0__py3-none-any.whl → 0.25.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of snowpark-connect might be problematic. Click here for more details.

Files changed (474) hide show
  1. snowflake/snowpark_connect/column_name_handler.py +116 -4
  2. snowflake/snowpark_connect/config.py +13 -0
  3. snowflake/snowpark_connect/constants.py +0 -29
  4. snowflake/snowpark_connect/dataframe_container.py +6 -0
  5. snowflake/snowpark_connect/execute_plan/map_execution_command.py +56 -1
  6. snowflake/snowpark_connect/expression/literal.py +13 -2
  7. snowflake/snowpark_connect/expression/map_cast.py +5 -8
  8. snowflake/snowpark_connect/expression/map_sql_expression.py +23 -1
  9. snowflake/snowpark_connect/expression/map_udf.py +26 -8
  10. snowflake/snowpark_connect/expression/map_unresolved_attribute.py +199 -15
  11. snowflake/snowpark_connect/expression/map_unresolved_extract_value.py +44 -16
  12. snowflake/snowpark_connect/expression/map_unresolved_function.py +825 -353
  13. snowflake/snowpark_connect/expression/map_unresolved_star.py +3 -2
  14. snowflake/snowpark_connect/hidden_column.py +39 -0
  15. snowflake/snowpark_connect/includes/jars/hadoop-client-api-trimmed-3.3.4.jar +0 -0
  16. snowflake/snowpark_connect/includes/jars/{hadoop-client-api-3.3.4.jar → spark-connect-client-jvm_2.12-3.5.6.jar} +0 -0
  17. snowflake/snowpark_connect/relation/map_column_ops.py +17 -4
  18. snowflake/snowpark_connect/relation/map_extension.py +52 -11
  19. snowflake/snowpark_connect/relation/map_join.py +258 -62
  20. snowflake/snowpark_connect/relation/map_sql.py +88 -11
  21. snowflake/snowpark_connect/relation/map_udtf.py +4 -2
  22. snowflake/snowpark_connect/relation/read/map_read.py +3 -3
  23. snowflake/snowpark_connect/relation/read/map_read_jdbc.py +1 -1
  24. snowflake/snowpark_connect/relation/read/map_read_json.py +8 -1
  25. snowflake/snowpark_connect/relation/read/map_read_table.py +1 -9
  26. snowflake/snowpark_connect/relation/read/reader_config.py +3 -1
  27. snowflake/snowpark_connect/relation/write/map_write.py +62 -53
  28. snowflake/snowpark_connect/resources_initializer.py +29 -1
  29. snowflake/snowpark_connect/server.py +18 -3
  30. snowflake/snowpark_connect/type_mapping.py +29 -25
  31. snowflake/snowpark_connect/typed_column.py +14 -0
  32. snowflake/snowpark_connect/utils/artifacts.py +23 -0
  33. snowflake/snowpark_connect/utils/context.py +6 -1
  34. snowflake/snowpark_connect/utils/scala_udf_utils.py +588 -0
  35. snowflake/snowpark_connect/utils/telemetry.py +6 -17
  36. snowflake/snowpark_connect/utils/udf_helper.py +2 -0
  37. snowflake/snowpark_connect/utils/udf_utils.py +38 -7
  38. snowflake/snowpark_connect/utils/udtf_utils.py +17 -3
  39. snowflake/snowpark_connect/version.py +1 -1
  40. {snowpark_connect-0.24.0.dist-info → snowpark_connect-0.25.0.dist-info}/METADATA +1 -1
  41. snowpark_connect-0.25.0.dist-info/RECORD +477 -0
  42. snowflake/snowpark_connect/includes/jars/scala-compiler-2.12.18.jar +0 -0
  43. snowflake/snowpark_connect/includes/jars/spark-kubernetes_2.12-3.5.6.jar +0 -0
  44. snowflake/snowpark_connect/includes/jars/spark-mllib_2.12-3.5.6.jar +0 -0
  45. snowflake/snowpark_connect/includes/jars/spark-streaming_2.12-3.5.6.jar +0 -0
  46. snowflake/snowpark_connect/includes/python/pyspark/errors/tests/__init__.py +0 -16
  47. snowflake/snowpark_connect/includes/python/pyspark/errors/tests/test_errors.py +0 -60
  48. snowflake/snowpark_connect/includes/python/pyspark/ml/deepspeed/tests/test_deepspeed_distributor.py +0 -306
  49. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/__init__.py +0 -16
  50. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_connect_classification.py +0 -53
  51. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_connect_evaluation.py +0 -50
  52. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_connect_feature.py +0 -43
  53. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_connect_function.py +0 -114
  54. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_connect_pipeline.py +0 -47
  55. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_connect_summarizer.py +0 -43
  56. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_connect_tuning.py +0 -46
  57. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_legacy_mode_classification.py +0 -238
  58. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_legacy_mode_evaluation.py +0 -194
  59. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_legacy_mode_feature.py +0 -156
  60. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_legacy_mode_pipeline.py +0 -184
  61. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_legacy_mode_summarizer.py +0 -78
  62. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_legacy_mode_tuning.py +0 -292
  63. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_parity_torch_data_loader.py +0 -50
  64. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_parity_torch_distributor.py +0 -152
  65. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_algorithms.py +0 -456
  66. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_base.py +0 -96
  67. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_dl_util.py +0 -186
  68. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_evaluation.py +0 -77
  69. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_feature.py +0 -401
  70. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_functions.py +0 -528
  71. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_image.py +0 -82
  72. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_linalg.py +0 -409
  73. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_model_cache.py +0 -55
  74. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_param.py +0 -441
  75. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_persistence.py +0 -546
  76. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_pipeline.py +0 -71
  77. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_stat.py +0 -52
  78. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_training_summary.py +0 -494
  79. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_util.py +0 -85
  80. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_wrapper.py +0 -138
  81. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/__init__.py +0 -16
  82. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/test_cv_io_basic.py +0 -151
  83. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/test_cv_io_nested.py +0 -97
  84. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/test_cv_io_pipeline.py +0 -143
  85. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/test_tuning.py +0 -551
  86. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/test_tvs_io_basic.py +0 -137
  87. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/test_tvs_io_nested.py +0 -96
  88. snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/test_tvs_io_pipeline.py +0 -142
  89. snowflake/snowpark_connect/includes/python/pyspark/ml/torch/tests/__init__.py +0 -16
  90. snowflake/snowpark_connect/includes/python/pyspark/ml/torch/tests/test_data_loader.py +0 -137
  91. snowflake/snowpark_connect/includes/python/pyspark/ml/torch/tests/test_distributor.py +0 -561
  92. snowflake/snowpark_connect/includes/python/pyspark/ml/torch/tests/test_log_communication.py +0 -172
  93. snowflake/snowpark_connect/includes/python/pyspark/mllib/tests/__init__.py +0 -16
  94. snowflake/snowpark_connect/includes/python/pyspark/mllib/tests/test_algorithms.py +0 -353
  95. snowflake/snowpark_connect/includes/python/pyspark/mllib/tests/test_feature.py +0 -192
  96. snowflake/snowpark_connect/includes/python/pyspark/mllib/tests/test_linalg.py +0 -680
  97. snowflake/snowpark_connect/includes/python/pyspark/mllib/tests/test_stat.py +0 -206
  98. snowflake/snowpark_connect/includes/python/pyspark/mllib/tests/test_streaming_algorithms.py +0 -471
  99. snowflake/snowpark_connect/includes/python/pyspark/mllib/tests/test_util.py +0 -108
  100. snowflake/snowpark_connect/includes/python/pyspark/pandas/spark/__init__.py +0 -16
  101. snowflake/snowpark_connect/includes/python/pyspark/pandas/spark/accessors.py +0 -1281
  102. snowflake/snowpark_connect/includes/python/pyspark/pandas/spark/functions.py +0 -203
  103. snowflake/snowpark_connect/includes/python/pyspark/pandas/spark/utils.py +0 -202
  104. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/__init__.py +0 -16
  105. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/__init__.py +0 -16
  106. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_any_all.py +0 -177
  107. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_apply_func.py +0 -575
  108. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_binary_ops.py +0 -235
  109. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_combine.py +0 -653
  110. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_compute.py +0 -463
  111. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_corrwith.py +0 -86
  112. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_cov.py +0 -151
  113. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_cumulative.py +0 -139
  114. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_describe.py +0 -458
  115. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_eval.py +0 -86
  116. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_melt.py +0 -202
  117. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_missing_data.py +0 -520
  118. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/computation/test_pivot.py +0 -361
  119. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/__init__.py +0 -16
  120. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/__init__.py +0 -16
  121. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_any_all.py +0 -40
  122. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_apply_func.py +0 -42
  123. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_binary_ops.py +0 -40
  124. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_combine.py +0 -37
  125. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_compute.py +0 -60
  126. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_corrwith.py +0 -40
  127. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_cov.py +0 -40
  128. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_cumulative.py +0 -90
  129. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_describe.py +0 -40
  130. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_eval.py +0 -40
  131. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_melt.py +0 -40
  132. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_missing_data.py +0 -42
  133. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/computation/test_parity_pivot.py +0 -37
  134. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/__init__.py +0 -16
  135. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_base.py +0 -36
  136. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_binary_ops.py +0 -42
  137. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_boolean_ops.py +0 -47
  138. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_categorical_ops.py +0 -55
  139. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_complex_ops.py +0 -40
  140. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_date_ops.py +0 -47
  141. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_datetime_ops.py +0 -47
  142. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_null_ops.py +0 -42
  143. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_num_arithmetic.py +0 -43
  144. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_num_ops.py +0 -47
  145. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_num_reverse.py +0 -43
  146. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_string_ops.py +0 -47
  147. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_timedelta_ops.py +0 -47
  148. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_udt_ops.py +0 -40
  149. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/data_type_ops/testing_utils.py +0 -226
  150. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/__init__.py +0 -16
  151. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_align.py +0 -39
  152. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_basic_slow.py +0 -55
  153. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_cov_corrwith.py +0 -39
  154. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_dot_frame.py +0 -39
  155. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_dot_series.py +0 -39
  156. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_index.py +0 -39
  157. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_series.py +0 -39
  158. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_setitem_frame.py +0 -43
  159. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_setitem_series.py +0 -43
  160. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/__init__.py +0 -16
  161. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_attrs.py +0 -40
  162. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_constructor.py +0 -39
  163. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_conversion.py +0 -42
  164. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_reindexing.py +0 -42
  165. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_reshaping.py +0 -37
  166. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_spark.py +0 -40
  167. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_take.py +0 -42
  168. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_time_series.py +0 -48
  169. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/frame/test_parity_truncate.py +0 -40
  170. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/__init__.py +0 -16
  171. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_aggregate.py +0 -40
  172. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_apply_func.py +0 -41
  173. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_cumulative.py +0 -67
  174. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_describe.py +0 -40
  175. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_groupby.py +0 -55
  176. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_head_tail.py +0 -40
  177. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_index.py +0 -38
  178. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_missing_data.py +0 -55
  179. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_split_apply.py +0 -39
  180. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/groupby/test_parity_stat.py +0 -38
  181. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/__init__.py +0 -16
  182. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_align.py +0 -40
  183. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_base.py +0 -50
  184. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_category.py +0 -73
  185. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_datetime.py +0 -39
  186. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_indexing.py +0 -40
  187. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_reindex.py +0 -40
  188. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_rename.py +0 -40
  189. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_reset_index.py +0 -48
  190. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/indexes/test_parity_timedelta.py +0 -39
  191. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/io/__init__.py +0 -16
  192. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/io/test_parity_io.py +0 -40
  193. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/plot/__init__.py +0 -16
  194. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot.py +0 -45
  195. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_matplotlib.py +0 -45
  196. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_plotly.py +0 -49
  197. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot.py +0 -37
  198. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_matplotlib.py +0 -53
  199. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_plotly.py +0 -45
  200. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/__init__.py +0 -16
  201. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_all_any.py +0 -38
  202. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_arg_ops.py +0 -37
  203. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_as_of.py +0 -37
  204. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_as_type.py +0 -38
  205. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_compute.py +0 -37
  206. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_conversion.py +0 -40
  207. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_cumulative.py +0 -40
  208. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_index.py +0 -38
  209. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_missing_data.py +0 -40
  210. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_series.py +0 -37
  211. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_sort.py +0 -38
  212. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/series/test_parity_stat.py +0 -38
  213. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_categorical.py +0 -66
  214. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_config.py +0 -37
  215. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_csv.py +0 -37
  216. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_dataframe_conversion.py +0 -42
  217. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_dataframe_spark_io.py +0 -39
  218. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_default_index.py +0 -49
  219. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_ewm.py +0 -37
  220. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_expanding.py +0 -39
  221. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_extension.py +0 -49
  222. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_frame_spark.py +0 -53
  223. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_generic_functions.py +0 -43
  224. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_indexing.py +0 -49
  225. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_indexops_spark.py +0 -39
  226. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_internal.py +0 -41
  227. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_namespace.py +0 -39
  228. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_numpy_compat.py +0 -60
  229. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_ops_on_diff_frames.py +0 -48
  230. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_ops_on_diff_frames_groupby.py +0 -39
  231. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_ops_on_diff_frames_groupby_expanding.py +0 -44
  232. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_ops_on_diff_frames_groupby_rolling.py +0 -84
  233. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_repr.py +0 -37
  234. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_resample.py +0 -45
  235. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_reshape.py +0 -39
  236. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_rolling.py +0 -39
  237. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_scalars.py +0 -37
  238. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_series_conversion.py +0 -39
  239. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_series_datetime.py +0 -39
  240. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_series_string.py +0 -39
  241. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_spark_functions.py +0 -39
  242. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_sql.py +0 -43
  243. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_stats.py +0 -37
  244. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_typedef.py +0 -36
  245. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_utils.py +0 -37
  246. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/connect/test_parity_window.py +0 -39
  247. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/__init__.py +0 -16
  248. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_base.py +0 -107
  249. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_binary_ops.py +0 -224
  250. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_boolean_ops.py +0 -825
  251. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_categorical_ops.py +0 -562
  252. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_complex_ops.py +0 -368
  253. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_date_ops.py +0 -257
  254. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_datetime_ops.py +0 -260
  255. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_null_ops.py +0 -178
  256. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_num_arithmetic.py +0 -184
  257. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_num_ops.py +0 -497
  258. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_num_reverse.py +0 -140
  259. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_string_ops.py +0 -354
  260. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_timedelta_ops.py +0 -219
  261. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/test_udt_ops.py +0 -192
  262. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/data_type_ops/testing_utils.py +0 -228
  263. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/__init__.py +0 -16
  264. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_align.py +0 -118
  265. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_basic_slow.py +0 -198
  266. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_cov_corrwith.py +0 -181
  267. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_dot_frame.py +0 -103
  268. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_dot_series.py +0 -141
  269. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_index.py +0 -109
  270. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_series.py +0 -136
  271. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_setitem_frame.py +0 -125
  272. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/diff_frames_ops/test_setitem_series.py +0 -217
  273. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/__init__.py +0 -16
  274. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_attrs.py +0 -384
  275. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_constructor.py +0 -598
  276. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_conversion.py +0 -73
  277. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_reindexing.py +0 -869
  278. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_reshaping.py +0 -487
  279. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_spark.py +0 -309
  280. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_take.py +0 -156
  281. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_time_series.py +0 -149
  282. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/frame/test_truncate.py +0 -163
  283. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/__init__.py +0 -16
  284. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_aggregate.py +0 -311
  285. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_apply_func.py +0 -524
  286. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_cumulative.py +0 -419
  287. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_describe.py +0 -144
  288. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_groupby.py +0 -979
  289. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_head_tail.py +0 -234
  290. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_index.py +0 -206
  291. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_missing_data.py +0 -421
  292. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_split_apply.py +0 -187
  293. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/groupby/test_stat.py +0 -397
  294. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/__init__.py +0 -16
  295. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_align.py +0 -100
  296. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_base.py +0 -2743
  297. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_category.py +0 -484
  298. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_datetime.py +0 -276
  299. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_indexing.py +0 -432
  300. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_reindex.py +0 -310
  301. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_rename.py +0 -257
  302. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_reset_index.py +0 -160
  303. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/indexes/test_timedelta.py +0 -128
  304. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/io/__init__.py +0 -16
  305. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/io/test_io.py +0 -137
  306. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/plot/__init__.py +0 -16
  307. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/plot/test_frame_plot.py +0 -170
  308. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/plot/test_frame_plot_matplotlib.py +0 -547
  309. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/plot/test_frame_plot_plotly.py +0 -285
  310. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/plot/test_series_plot.py +0 -106
  311. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/plot/test_series_plot_matplotlib.py +0 -409
  312. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/plot/test_series_plot_plotly.py +0 -247
  313. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/__init__.py +0 -16
  314. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_all_any.py +0 -105
  315. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_arg_ops.py +0 -197
  316. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_as_of.py +0 -137
  317. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_as_type.py +0 -227
  318. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_compute.py +0 -634
  319. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_conversion.py +0 -88
  320. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_cumulative.py +0 -139
  321. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_index.py +0 -475
  322. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_missing_data.py +0 -265
  323. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_series.py +0 -818
  324. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_sort.py +0 -162
  325. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/series/test_stat.py +0 -780
  326. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_categorical.py +0 -741
  327. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_config.py +0 -160
  328. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_csv.py +0 -453
  329. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_dataframe_conversion.py +0 -281
  330. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_dataframe_spark_io.py +0 -487
  331. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_default_index.py +0 -109
  332. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_ewm.py +0 -434
  333. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_expanding.py +0 -253
  334. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_extension.py +0 -152
  335. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_frame_spark.py +0 -162
  336. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_generic_functions.py +0 -234
  337. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_indexing.py +0 -1339
  338. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_indexops_spark.py +0 -82
  339. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_internal.py +0 -124
  340. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_namespace.py +0 -638
  341. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_numpy_compat.py +0 -200
  342. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_ops_on_diff_frames.py +0 -1355
  343. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby.py +0 -655
  344. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_expanding.py +0 -113
  345. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_rolling.py +0 -118
  346. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_repr.py +0 -192
  347. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_resample.py +0 -346
  348. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_reshape.py +0 -495
  349. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_rolling.py +0 -263
  350. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_scalars.py +0 -59
  351. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_series_conversion.py +0 -85
  352. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_series_datetime.py +0 -364
  353. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_series_string.py +0 -362
  354. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_spark_functions.py +0 -46
  355. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_sql.py +0 -123
  356. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_stats.py +0 -581
  357. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_typedef.py +0 -447
  358. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_utils.py +0 -301
  359. snowflake/snowpark_connect/includes/python/pyspark/pandas/tests/test_window.py +0 -465
  360. snowflake/snowpark_connect/includes/python/pyspark/resource/tests/__init__.py +0 -16
  361. snowflake/snowpark_connect/includes/python/pyspark/resource/tests/test_resources.py +0 -83
  362. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/__init__.py +0 -16
  363. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/__init__.py +0 -16
  364. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/client/__init__.py +0 -16
  365. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/client/test_artifact.py +0 -420
  366. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/client/test_client.py +0 -358
  367. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/streaming/__init__.py +0 -16
  368. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/streaming/test_parity_foreach.py +0 -36
  369. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/streaming/test_parity_foreach_batch.py +0 -44
  370. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/streaming/test_parity_listener.py +0 -116
  371. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/streaming/test_parity_streaming.py +0 -35
  372. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_connect_basic.py +0 -3612
  373. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_connect_column.py +0 -1042
  374. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_connect_function.py +0 -2381
  375. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_connect_plan.py +0 -1060
  376. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_arrow.py +0 -163
  377. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_arrow_map.py +0 -38
  378. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_arrow_python_udf.py +0 -48
  379. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_catalog.py +0 -36
  380. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_column.py +0 -55
  381. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_conf.py +0 -36
  382. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_dataframe.py +0 -96
  383. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_datasources.py +0 -44
  384. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_errors.py +0 -36
  385. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_functions.py +0 -59
  386. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_group.py +0 -36
  387. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_pandas_cogrouped_map.py +0 -59
  388. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_pandas_grouped_map.py +0 -74
  389. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_pandas_grouped_map_with_state.py +0 -62
  390. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_pandas_map.py +0 -58
  391. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_pandas_udf.py +0 -70
  392. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_pandas_udf_grouped_agg.py +0 -50
  393. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_pandas_udf_scalar.py +0 -68
  394. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_pandas_udf_window.py +0 -40
  395. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_readwriter.py +0 -46
  396. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_serde.py +0 -44
  397. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_types.py +0 -100
  398. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_udf.py +0 -100
  399. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_udtf.py +0 -163
  400. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_session.py +0 -181
  401. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_utils.py +0 -42
  402. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/__init__.py +0 -16
  403. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py +0 -623
  404. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_grouped_map.py +0 -869
  405. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_grouped_map_with_state.py +0 -342
  406. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_map.py +0 -436
  407. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_udf.py +0 -363
  408. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_udf_grouped_agg.py +0 -592
  409. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_udf_scalar.py +0 -1503
  410. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_udf_typehints.py +0 -392
  411. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_udf_typehints_with_future_annotations.py +0 -375
  412. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/test_pandas_udf_window.py +0 -411
  413. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/streaming/__init__.py +0 -16
  414. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/streaming/test_streaming.py +0 -401
  415. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/streaming/test_streaming_foreach.py +0 -295
  416. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/streaming/test_streaming_foreach_batch.py +0 -106
  417. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/streaming/test_streaming_listener.py +0 -558
  418. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_arrow.py +0 -1346
  419. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_arrow_map.py +0 -182
  420. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_arrow_python_udf.py +0 -202
  421. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_catalog.py +0 -503
  422. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_column.py +0 -225
  423. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_conf.py +0 -83
  424. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_context.py +0 -201
  425. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_dataframe.py +0 -1931
  426. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_datasources.py +0 -256
  427. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_errors.py +0 -69
  428. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_functions.py +0 -1349
  429. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_group.py +0 -53
  430. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_pandas_sqlmetrics.py +0 -68
  431. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_readwriter.py +0 -283
  432. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_serde.py +0 -155
  433. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_session.py +0 -412
  434. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_types.py +0 -1581
  435. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_udf.py +0 -961
  436. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_udf_profiler.py +0 -165
  437. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_udtf.py +0 -1456
  438. snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_utils.py +0 -1686
  439. snowflake/snowpark_connect/includes/python/pyspark/streaming/tests/__init__.py +0 -16
  440. snowflake/snowpark_connect/includes/python/pyspark/streaming/tests/test_context.py +0 -184
  441. snowflake/snowpark_connect/includes/python/pyspark/streaming/tests/test_dstream.py +0 -706
  442. snowflake/snowpark_connect/includes/python/pyspark/streaming/tests/test_kinesis.py +0 -118
  443. snowflake/snowpark_connect/includes/python/pyspark/streaming/tests/test_listener.py +0 -160
  444. snowflake/snowpark_connect/includes/python/pyspark/tests/__init__.py +0 -16
  445. snowflake/snowpark_connect/includes/python/pyspark/tests/test_appsubmit.py +0 -306
  446. snowflake/snowpark_connect/includes/python/pyspark/tests/test_broadcast.py +0 -196
  447. snowflake/snowpark_connect/includes/python/pyspark/tests/test_conf.py +0 -44
  448. snowflake/snowpark_connect/includes/python/pyspark/tests/test_context.py +0 -346
  449. snowflake/snowpark_connect/includes/python/pyspark/tests/test_daemon.py +0 -89
  450. snowflake/snowpark_connect/includes/python/pyspark/tests/test_install_spark.py +0 -124
  451. snowflake/snowpark_connect/includes/python/pyspark/tests/test_join.py +0 -69
  452. snowflake/snowpark_connect/includes/python/pyspark/tests/test_memory_profiler.py +0 -167
  453. snowflake/snowpark_connect/includes/python/pyspark/tests/test_pin_thread.py +0 -194
  454. snowflake/snowpark_connect/includes/python/pyspark/tests/test_profiler.py +0 -168
  455. snowflake/snowpark_connect/includes/python/pyspark/tests/test_rdd.py +0 -939
  456. snowflake/snowpark_connect/includes/python/pyspark/tests/test_rddbarrier.py +0 -52
  457. snowflake/snowpark_connect/includes/python/pyspark/tests/test_rddsampler.py +0 -66
  458. snowflake/snowpark_connect/includes/python/pyspark/tests/test_readwrite.py +0 -368
  459. snowflake/snowpark_connect/includes/python/pyspark/tests/test_serializers.py +0 -257
  460. snowflake/snowpark_connect/includes/python/pyspark/tests/test_shuffle.py +0 -267
  461. snowflake/snowpark_connect/includes/python/pyspark/tests/test_stage_sched.py +0 -153
  462. snowflake/snowpark_connect/includes/python/pyspark/tests/test_statcounter.py +0 -130
  463. snowflake/snowpark_connect/includes/python/pyspark/tests/test_taskcontext.py +0 -350
  464. snowflake/snowpark_connect/includes/python/pyspark/tests/test_util.py +0 -97
  465. snowflake/snowpark_connect/includes/python/pyspark/tests/test_worker.py +0 -271
  466. snowpark_connect-0.24.0.dist-info/RECORD +0 -898
  467. {snowpark_connect-0.24.0.data → snowpark_connect-0.25.0.data}/scripts/snowpark-connect +0 -0
  468. {snowpark_connect-0.24.0.data → snowpark_connect-0.25.0.data}/scripts/snowpark-session +0 -0
  469. {snowpark_connect-0.24.0.data → snowpark_connect-0.25.0.data}/scripts/snowpark-submit +0 -0
  470. {snowpark_connect-0.24.0.dist-info → snowpark_connect-0.25.0.dist-info}/WHEEL +0 -0
  471. {snowpark_connect-0.24.0.dist-info → snowpark_connect-0.25.0.dist-info}/licenses/LICENSE-binary +0 -0
  472. {snowpark_connect-0.24.0.dist-info → snowpark_connect-0.25.0.dist-info}/licenses/LICENSE.txt +0 -0
  473. {snowpark_connect-0.24.0.dist-info → snowpark_connect-0.25.0.dist-info}/licenses/NOTICE-binary +0 -0
  474. {snowpark_connect-0.24.0.dist-info → snowpark_connect-0.25.0.dist-info}/top_level.txt +0 -0
@@ -73,11 +73,11 @@ from snowflake.snowpark_connect.column_name_handler import (
73
73
  )
74
74
  from snowflake.snowpark_connect.config import (
75
75
  get_boolean_session_config_param,
76
+ get_timestamp_type,
76
77
  global_config,
77
78
  )
78
79
  from snowflake.snowpark_connect.constants import (
79
80
  DUPLICATE_KEY_FOUND_ERROR_TEMPLATE,
80
- SPARK_TZ_ABBREVIATIONS_OVERRIDES,
81
81
  STRUCTURED_TYPES_ENABLED,
82
82
  )
83
83
  from snowflake.snowpark_connect.expression.function_defaults import (
@@ -100,7 +100,6 @@ from snowflake.snowpark_connect.type_mapping import (
100
100
  map_snowpark_to_pyspark_types,
101
101
  map_spark_timestamp_format_expression,
102
102
  map_type_string_to_snowpark_type,
103
- parse_ddl_string,
104
103
  )
105
104
  from snowflake.snowpark_connect.typed_column import (
106
105
  TypedColumn,
@@ -209,19 +208,6 @@ def _validate_numeric_args(
209
208
  return modified_args
210
209
 
211
210
 
212
- def get_timestamp_type():
213
- match global_config["spark.sql.timestampType"]:
214
- case "TIMESTAMP_LTZ":
215
- timestamp_type = TimestampType(TimestampTimeZone.LTZ)
216
- case "TIMESTAMP_NTZ":
217
- timestamp_type = TimestampType(TimestampTimeZone.NTZ)
218
- case "TIMESTAMP_TZ":
219
- timestamp_type = TimestampType(TimestampTimeZone.TZ)
220
- case _:
221
- timestamp_type = TimestampType(TimestampTimeZone.DEFAULT)
222
- return timestamp_type
223
-
224
-
225
211
  def unwrap_literal(exp: expressions_proto.Expression):
226
212
  """Workaround for Snowpark functions generating invalid SQL when used with fn.lit (SNOW-1871954)"""
227
213
  return get_literal_field_and_name(exp.literal)[0]
@@ -522,146 +508,401 @@ def map_unresolved_function(
522
508
  result_exp = snowpark_fn.when(
523
509
  snowpark_args[1] == 0, snowpark_fn.lit(None)
524
510
  ).otherwise(snowpark_args[0] % snowpark_args[1])
525
- result_type = _find_common_type([arg.typ for arg in snowpark_typed_args])
526
- case "*":
527
- # For decimal arithmetic, we need to check for overflow
528
- if (
529
- isinstance(snowpark_typed_args[0].typ, DecimalType)
530
- and isinstance(snowpark_typed_args[1].typ, DecimalType)
531
- or isinstance(snowpark_typed_args[0].typ, DecimalType)
532
- and isinstance(snowpark_typed_args[1].typ, _IntegralType)
533
- or isinstance(snowpark_typed_args[0].typ, _IntegralType)
534
- and isinstance(snowpark_typed_args[1].typ, DecimalType)
535
- ):
536
- result_exp, (
537
- return_type_precision,
538
- return_type_scale,
539
- ) = _mul_div_precision_helper(snowpark_typed_args, snowpark_args, 0)
540
- result_type = DecimalType(return_type_precision, return_type_scale)
541
- else:
542
- result_exp = snowpark_args[0] * snowpark_args[1]
543
- result_type = _find_common_type(
544
- [arg.typ for arg in snowpark_typed_args]
545
- )
546
- case "+":
547
- if isinstance(snowpark_typed_args[0].typ, DateType) and not isinstance(
548
- snowpark_typed_args[1].typ,
549
- (_IntegralType, StringType),
550
- ):
551
- raise AnalysisException(
552
- f'[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] Cannot resolve "{snowpark_arg_names[0]} + {snowpark_arg_names[1]}" due to data type mismatch: Parameter 2 requires the ("INT" or "SMALLINT" or "TINYINT") type, however "{snowpark_arg_names[1]}" has the type "{snowpark_typed_args[1].typ}".',
553
- )
554
- elif (
555
- isinstance(snowpark_typed_args[0].typ, BinaryType)
556
- and isinstance(snowpark_typed_args[1].typ, DecimalType)
557
- or isinstance(snowpark_typed_args[0].typ, DecimalType)
558
- and isinstance(snowpark_typed_args[1].typ, BinaryType)
559
- ) or (
560
- isinstance(snowpark_typed_args[0].typ, DecimalType)
561
- and isinstance(snowpark_typed_args[1].typ, TimestampType)
562
- or isinstance(snowpark_typed_args[0].typ, TimestampType)
563
- and isinstance(snowpark_typed_args[1].typ, DecimalType)
564
- ):
565
- raise AnalysisException(
566
- f'[DATATYPE_MISMATCH.BINARY_OP_DIFF_TYPES] Cannot resolve "({snowpark_arg_names[0]} + {snowpark_arg_names[1]})" due to data type mismatch: the left and right operands of the binary operator have incompatible types ("{snowpark_typed_args[0].typ}" and "{snowpark_typed_args[1].typ}").'
567
- )
568
- elif isinstance(snowpark_typed_args[1].typ, DateType) and not isinstance(
569
- snowpark_typed_args[0].typ,
570
- (_IntegralType, StringType),
571
- ):
572
- raise AnalysisException(
573
- f'[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] Cannot resolve "{snowpark_arg_names[0]} + {snowpark_arg_names[1]}" due to data type mismatch: Parameter 1 requires the ("INT" or "SMALLINT" or "TINYINT") type, however "{snowpark_arg_names[0]}" has the type "{snowpark_typed_args[0].typ}".',
574
- )
575
- elif (
576
- isinstance(snowpark_typed_args[0].typ, DecimalType)
577
- and isinstance(snowpark_typed_args[1].typ, DecimalType)
578
- or isinstance(snowpark_typed_args[0].typ, DecimalType)
579
- and isinstance(snowpark_typed_args[1].typ, _IntegralType)
580
- or isinstance(snowpark_typed_args[0].typ, _IntegralType)
581
- and isinstance(snowpark_typed_args[1].typ, DecimalType)
582
- ):
583
- result_exp, (
584
- return_type_precision,
585
- return_type_scale,
586
- ) = _add_sub_precision_helper(snowpark_typed_args, snowpark_args, 0)
587
- result_type = DecimalType(return_type_precision, return_type_scale)
588
- else:
589
- result_exp = snowpark_args[0] + snowpark_args[1]
590
- if any(isinstance(arg.typ, DateType) for arg in snowpark_typed_args):
591
- # TODO SNOW-2034420: resolve return type
592
- result_exp = _type_with_typer(result_exp)
593
- else:
511
+ match (snowpark_typed_args[0].typ, snowpark_typed_args[1].typ):
512
+ case (NullType(), NullType()):
513
+ result_type = DoubleType()
514
+ case _:
594
515
  result_type = _find_common_type(
595
516
  [arg.typ for arg in snowpark_typed_args]
596
517
  )
597
- case "-":
598
- if isinstance(snowpark_typed_args[0].typ, DateType) and not isinstance(
599
- snowpark_typed_args[1].typ,
600
- (_IntegralType, StringType),
601
- ):
602
- raise AnalysisException(
603
- f'[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] Cannot resolve "{snowpark_arg_names[0]} - {snowpark_arg_names[1]}" due to data type mismatch: Parameter 2 requires the ("INT" or "SMALLINT" or "TINYINT") type, however "{snowpark_arg_names[1]}" has the type "{snowpark_typed_args[1].typ}".',
604
- )
605
- elif (
606
- isinstance(snowpark_typed_args[0].typ, BinaryType)
607
- and isinstance(snowpark_typed_args[1].typ, DecimalType)
608
- or isinstance(snowpark_typed_args[0].typ, DecimalType)
609
- and isinstance(snowpark_typed_args[1].typ, BinaryType)
610
- ) or (
611
- isinstance(snowpark_typed_args[0].typ, DecimalType)
612
- and isinstance(snowpark_typed_args[1].typ, TimestampType)
613
- or isinstance(snowpark_typed_args[0].typ, TimestampType)
614
- and isinstance(snowpark_typed_args[1].typ, DecimalType)
615
- ):
616
- raise AnalysisException(
617
- f'[DATATYPE_MISMATCH.BINARY_OP_DIFF_TYPES] Cannot resolve "({snowpark_arg_names[0]} - {snowpark_arg_names[1]})" due to data type mismatch: the left and right operands of the binary operator have incompatible types ("{snowpark_typed_args[0].typ}" and "{snowpark_typed_args[1].typ}").'
618
- )
619
- elif isinstance(snowpark_typed_args[1].typ, DateType) and not isinstance(
620
- snowpark_typed_args[0].typ,
621
- (_IntegralType, StringType),
622
- ):
623
- raise AnalysisException(
624
- f'[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] Cannot resolve "{snowpark_arg_names[0]} - {snowpark_arg_names[1]}" due to data type mismatch: Parameter 1 requires the ("INT" or "SMALLINT" or "TINYINT") type, however "{snowpark_arg_names[0]}" has the type "{snowpark_typed_args[0].typ}".',
625
- )
626
- elif (
627
- isinstance(snowpark_typed_args[0].typ, DecimalType)
628
- and isinstance(snowpark_typed_args[1].typ, DecimalType)
629
- or isinstance(snowpark_typed_args[0].typ, DecimalType)
630
- and isinstance(snowpark_typed_args[1].typ, _IntegralType)
631
- or isinstance(snowpark_typed_args[0].typ, _IntegralType)
632
- and isinstance(snowpark_typed_args[1].typ, DecimalType)
633
- ):
634
- result_exp, (
635
- return_type_precision,
636
- return_type_scale,
637
- ) = _add_sub_precision_helper(snowpark_typed_args, snowpark_args, 1)
638
- result_type = DecimalType(return_type_precision, return_type_scale)
639
- else:
640
- result_exp = snowpark_args[0] - snowpark_args[1]
641
- if any(isinstance(arg.typ, DateType) for arg in snowpark_typed_args):
642
- # TODO SNOW-2034420: resolve return type
643
- result_exp = _type_with_typer(result_exp)
644
- else:
518
+ case "*":
519
+ match (snowpark_typed_args[0].typ, snowpark_typed_args[1].typ):
520
+ case (DecimalType() as t, NullType()) | (
521
+ NullType(),
522
+ DecimalType() as t,
523
+ ):
524
+ p1, s1 = _get_type_precision(t)
525
+ result_type = _get_decimal_multiplication_result_type(
526
+ p1, s1, p1, s1
527
+ )
528
+ result_exp = snowpark_fn.lit(None)
529
+ case (DecimalType(), t) | (t, DecimalType()) if isinstance(
530
+ t, (DecimalType, _IntegralType)
531
+ ):
532
+ p1, s1 = _get_type_precision(snowpark_typed_args[0].typ)
533
+ p2, s2 = _get_type_precision(snowpark_typed_args[1].typ)
534
+ result_type = _get_decimal_multiplication_result_type(
535
+ p1, s1, p2, s2
536
+ )
537
+ result_exp = _get_decimal_multiplication_result_exp(
538
+ result_type, t, snowpark_args
539
+ )
540
+ case (NullType(), NullType()):
541
+ result_type = DoubleType()
542
+ result_exp = snowpark_fn.lit(None)
543
+ case (StringType(), StringType()):
544
+ if spark_sql_ansi_enabled:
545
+ raise AnalysisException(
546
+ f'[DATATYPE_MISMATCH.BINARY_OP_WRONG_TYPE] Cannot resolve "{spark_function_name}" due to data type mismatch: the binary operator requires the input type ("DOUBLE" or "DECIMAL"), not "STRING".'
547
+ )
548
+ else:
549
+ result_type = DoubleType()
550
+ result_exp = snowpark_args[0].try_cast(
551
+ result_type
552
+ ) * snowpark_args[1].try_cast(result_type)
553
+ case (StringType(), _IntegralType()):
554
+ if spark_sql_ansi_enabled:
555
+ result_type = LongType()
556
+ result_exp = (
557
+ snowpark_args[0].cast(result_type) * snowpark_args[1]
558
+ )
559
+ else:
560
+ result_type = DoubleType()
561
+ result_exp = (
562
+ snowpark_args[0].try_cast(result_type) * snowpark_args[1]
563
+ )
564
+ case (StringType(), _FractionalType()):
565
+ result_type = DoubleType()
566
+ if spark_sql_ansi_enabled:
567
+ result_exp = (
568
+ snowpark_args[0].cast(result_type) * snowpark_args[1]
569
+ )
570
+ else:
571
+ result_exp = (
572
+ snowpark_args[0].try_cast(result_type) * snowpark_args[1]
573
+ )
574
+ case (_IntegralType(), StringType()):
575
+ if spark_sql_ansi_enabled:
576
+ result_type = LongType()
577
+ result_exp = snowpark_args[0] * snowpark_args[1].cast(
578
+ result_type
579
+ )
580
+ else:
581
+ result_type = DoubleType()
582
+ result_exp = snowpark_args[0] * snowpark_args[1].try_cast(
583
+ result_type
584
+ )
585
+ case (_FractionalType(), StringType()):
586
+ result_type = DoubleType()
587
+ if spark_sql_ansi_enabled:
588
+ result_exp = snowpark_args[0] * snowpark_args[1].cast(
589
+ result_type
590
+ )
591
+ else:
592
+ result_exp = snowpark_args[0] * snowpark_args[1].try_cast(
593
+ result_type
594
+ )
595
+ case (_NumericType() as t, NullType()) | (
596
+ NullType(),
597
+ _NumericType() as t,
598
+ ):
599
+ result_type = t
600
+ result_exp = snowpark_fn.lit(None)
601
+ case (_NumericType(), _NumericType()):
645
602
  result_type = _find_common_type(
646
603
  [arg.typ for arg in snowpark_typed_args]
647
604
  )
648
- case "/":
605
+ result_exp = snowpark_args[0].cast(result_type) * snowpark_args[
606
+ 1
607
+ ].cast(result_type)
608
+ case _:
609
+ raise AnalysisException(
610
+ f'[DATATYPE_MISMATCH.BINARY_OP_DIFF_TYPES] Cannot resolve "{spark_function_name}" due to data type mismatch: the left and right operands of the binary operator have incompatible types ("{snowpark_typed_args[0].typ}" and "{snowpark_typed_args[1].typ}").'
611
+ )
612
+ case "+":
613
+ spark_function_name = _get_spark_function_name(
614
+ snowpark_typed_args[0],
615
+ snowpark_typed_args[1],
616
+ snowpark_arg_names,
617
+ exp,
618
+ spark_function_name,
619
+ "+",
620
+ )
649
621
  match (snowpark_typed_args[0].typ, snowpark_typed_args[1].typ):
622
+ case (NullType(), _) | (_, NullType()):
623
+ result_type = _get_add_sub_result_type(
624
+ snowpark_typed_args[0].typ,
625
+ snowpark_typed_args[1].typ,
626
+ spark_function_name,
627
+ )
628
+ result_exp = snowpark_args[0] + snowpark_args[1]
629
+ result_exp = result_exp.cast(result_type)
630
+ case (DateType(), t) | (t, DateType()):
631
+ date_param_index = (
632
+ 0 if isinstance(snowpark_typed_args[0].typ, DateType) else 1
633
+ )
634
+ t_param_index = 1 - date_param_index
635
+ if isinstance(t, (IntegerType, ShortType, ByteType)):
636
+ result_type = DateType()
637
+ result_exp = snowpark_args[0] + snowpark_args[1]
638
+ elif (
639
+ "INTERVAL"
640
+ == snowpark_typed_args[t_param_index].col._expr1.pretty_name
641
+ ):
642
+ result_type = TimestampType()
643
+ result_exp = (
644
+ snowpark_args[date_param_index]
645
+ + snowpark_args[t_param_index]
646
+ )
647
+ else:
648
+ raise AnalysisException(
649
+ f'[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] Cannot resolve "{spark_function_name}" due to data type mismatch: Parameter 2 requires the ("INT" or "SMALLINT" or "TINYINT") type, however "{snowpark_arg_names[t_param_index]}" has the type "{t}".',
650
+ )
651
+ case (StringType(), StringType()):
652
+ if spark_sql_ansi_enabled:
653
+ raise AnalysisException(
654
+ f'[DATATYPE_MISMATCH.BINARY_OP_WRONG_TYPE] Cannot resolve "{spark_function_name}" due to data type mismatch: the binary operator requires the input type ("NUMERIC" or "INTERVAL DAY TO SECOND" or "INTERVAL YEAR TO MONTH" or "INTERVAL"), not "STRING".'
655
+ )
656
+ else:
657
+ result_type = DoubleType()
658
+ result_exp = snowpark_fn.try_cast(
659
+ snowpark_args[0], result_type
660
+ ) + snowpark_fn.try_cast(snowpark_args[1], result_type)
661
+ case (StringType(), _NumericType() as t):
662
+ if spark_sql_ansi_enabled:
663
+ result_type = (
664
+ DoubleType()
665
+ if isinstance(t, _FractionalType)
666
+ else LongType()
667
+ )
668
+ result_exp = (
669
+ snowpark_args[0].cast(result_type) + snowpark_args[1]
670
+ )
671
+ else:
672
+ result_type = DoubleType()
673
+ result_exp = (
674
+ snowpark_fn.try_cast(snowpark_args[0], result_type)
675
+ + snowpark_args[1]
676
+ )
677
+ case (_NumericType() as t, StringType()):
678
+ if spark_sql_ansi_enabled:
679
+ result_type = (
680
+ DoubleType()
681
+ if isinstance(t, _FractionalType)
682
+ else LongType()
683
+ )
684
+ result_exp = snowpark_args[0] + snowpark_args[1].cast(
685
+ result_type
686
+ )
687
+ else:
688
+ result_type = DoubleType()
689
+ result_exp = snowpark_args[0] + snowpark_fn.try_cast(
690
+ snowpark_args[1], result_type
691
+ )
650
692
  case (DecimalType(), t) | (t, DecimalType()) if isinstance(
651
- t, DecimalType
652
- ) or isinstance(t, _IntegralType) or isinstance(
653
- snowpark_typed_args[1].typ, NullType
693
+ t, (BinaryType, TimestampType)
654
694
  ):
655
- result_exp, (
656
- return_type_precision,
657
- return_type_scale,
658
- ) = _mul_div_precision_helper(snowpark_typed_args, snowpark_args, 1)
659
- result_type = DecimalType(return_type_precision, return_type_scale)
695
+ raise AnalysisException(
696
+ f'[DATATYPE_MISMATCH.BINARY_OP_DIFF_TYPES] Cannot resolve "{spark_function_name}" due to data type mismatch: the left and right operands of the binary operator have incompatible types ("{snowpark_typed_args[0].typ}" and "{snowpark_typed_args[1].typ}").'
697
+ )
660
698
  case _:
699
+ result_type = _get_add_sub_result_type(
700
+ snowpark_typed_args[0].typ,
701
+ snowpark_typed_args[1].typ,
702
+ spark_function_name,
703
+ )
704
+ result_exp = snowpark_args[0] + snowpark_args[1]
705
+ if isinstance(result_type, DecimalType):
706
+ result_exp = _cast_helper(result_exp, result_type)
707
+ case "-":
708
+ spark_function_name = _get_spark_function_name(
709
+ snowpark_typed_args[0],
710
+ snowpark_typed_args[1],
711
+ snowpark_arg_names,
712
+ exp,
713
+ spark_function_name,
714
+ "-",
715
+ )
716
+ match (snowpark_typed_args[0].typ, snowpark_typed_args[1].typ):
717
+ case (NullType(), _) | (_, NullType()):
718
+ result_type = _get_add_sub_result_type(
719
+ snowpark_typed_args[0].typ,
720
+ snowpark_typed_args[1].typ,
721
+ spark_function_name,
722
+ )
723
+ result_exp = snowpark_args[0] - snowpark_args[1]
724
+ result_exp = result_exp.cast(result_type)
725
+ case (DateType(), DateType()):
726
+ # TODO SNOW-2034420: resolve return type (it should be INTERVAL DAY)
727
+ result_type = LongType()
728
+ result_exp = snowpark_args[0] - snowpark_args[1]
729
+ case (DateType(), StringType()):
730
+ if "INTERVAL" == snowpark_typed_args[1].col._expr1.pretty_name:
731
+ result_type = TimestampType()
732
+ result_exp = snowpark_args[0] - snowpark_args[1]
733
+ else:
734
+ # TODO SNOW-2034420: resolve return type (it should be INTERVAL DAY)
735
+ result_type = LongType()
736
+ input_type = (
737
+ DateType() if spark_sql_ansi_enabled else DoubleType()
738
+ )
739
+ result_exp = snowpark_args[0] - snowpark_args[1].cast(
740
+ input_type
741
+ )
742
+ case (StringType(), DateType()):
743
+ # TODO SNOW-2034420: resolve return type (it should be INTERVAL DAY)
744
+ result_type = LongType()
745
+ input_type = DateType()
746
+ result_exp = snowpark_args[0].cast(input_type) - snowpark_args[1]
747
+ case (DateType(), (IntegerType() | ShortType() | ByteType())):
748
+ result_type = DateType()
749
+ result_exp = snowpark_args[0] - snowpark_args[1]
750
+ case (DateType(), _):
751
+ raise AnalysisException(
752
+ f'[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] Cannot resolve "{spark_function_name}" due to data type mismatch: Parameter 2 requires the ("INT" or "SMALLINT" or "TINYINT") type, however "{snowpark_arg_names[1]}" has the type "{snowpark_typed_args[1].typ}".',
753
+ )
754
+ case (_, DateType()):
755
+ raise AnalysisException(
756
+ f'[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] Cannot resolve "{spark_function_name}" due to data type mismatch: Parameter 1 requires the "DATE" type, however "{snowpark_arg_names[0]}" has the type "{snowpark_typed_args[0].typ}".',
757
+ )
758
+ case (StringType(), StringType()):
759
+ if spark_sql_ansi_enabled:
760
+ raise AnalysisException(
761
+ f'[DATATYPE_MISMATCH.BINARY_OP_WRONG_TYPE] Cannot resolve "{spark_function_name}" due to data type mismatch: the binary operator requires the input type ("NUMERIC" or "INTERVAL DAY TO SECOND" or "INTERVAL YEAR TO MONTH" or "INTERVAL"), not "STRING".'
762
+ )
763
+ else:
764
+ result_type = DoubleType()
765
+ result_exp = snowpark_fn.try_cast(
766
+ snowpark_args[0], result_type
767
+ ) - snowpark_fn.try_cast(snowpark_args[1], result_type)
768
+ case (StringType(), _NumericType() as t):
769
+ if spark_sql_ansi_enabled:
770
+ result_type = (
771
+ DoubleType()
772
+ if isinstance(t, _FractionalType)
773
+ else LongType()
774
+ )
775
+ result_exp = (
776
+ snowpark_args[0].cast(result_type) - snowpark_args[1]
777
+ )
778
+ else:
779
+ result_type = DoubleType()
780
+ result_exp = (
781
+ snowpark_fn.try_cast(snowpark_args[0], result_type)
782
+ - snowpark_args[1]
783
+ )
784
+ case (_NumericType() as t, StringType()):
785
+ if spark_sql_ansi_enabled:
786
+ result_type = (
787
+ DoubleType()
788
+ if isinstance(t, _FractionalType)
789
+ else LongType()
790
+ )
791
+ result_exp = snowpark_args[0] - snowpark_args[1].cast(
792
+ result_type
793
+ )
794
+ else:
795
+ result_type = DoubleType()
796
+ result_exp = snowpark_args[0] - snowpark_fn.try_cast(
797
+ snowpark_args[1], result_type
798
+ )
799
+ case (DecimalType(), t) | (t, DecimalType()) if isinstance(
800
+ t, (BinaryType, TimestampType)
801
+ ):
802
+ raise AnalysisException(
803
+ f'[DATATYPE_MISMATCH.BINARY_OP_DIFF_TYPES] Cannot resolve "{spark_function_name}" due to data type mismatch: the left and right operands of the binary operator have incompatible types ("{snowpark_typed_args[0].typ}" and "{snowpark_typed_args[1].typ}").'
804
+ )
805
+ case _:
806
+ result_type = _get_add_sub_result_type(
807
+ snowpark_typed_args[0].typ,
808
+ snowpark_typed_args[1].typ,
809
+ spark_function_name,
810
+ )
811
+ result_exp = snowpark_args[0] - snowpark_args[1]
812
+ if isinstance(result_type, DecimalType):
813
+ result_exp = _cast_helper(result_exp, result_type)
814
+ case "/":
815
+ match (snowpark_typed_args[0].typ, snowpark_typed_args[1].typ):
816
+ case (DecimalType() as t1, NullType()):
817
+ p1, s1 = _get_type_precision(t1)
818
+ result_type, _ = _get_decimal_division_result_type(p1, s1, p1, s1)
819
+ result_exp = snowpark_fn.lit(None).cast(result_type)
820
+ case (DecimalType(), t) | (t, DecimalType()) if isinstance(
821
+ t, (DecimalType, _IntegralType)
822
+ ):
823
+ p1, s1 = _get_type_precision(snowpark_typed_args[0].typ)
824
+ p2, s2 = _get_type_precision(snowpark_typed_args[1].typ)
825
+ result_type, overflow_detected = _get_decimal_division_result_type(
826
+ p1, s1, p2, s2
827
+ )
828
+ result_exp = _get_decimal_division_result_exp(
829
+ result_type,
830
+ t,
831
+ overflow_detected,
832
+ snowpark_args,
833
+ spark_function_name,
834
+ )
835
+ case (NullType(), NullType()):
661
836
  result_type = DoubleType()
662
- dividend = snowpark_args[0].cast(result_type)
663
- divisor = snowpark_args[1].cast(result_type)
664
- result_exp = _divnull(dividend, divisor)
837
+ result_exp = snowpark_fn.lit(None)
838
+ case (StringType(), StringType()):
839
+ if spark_sql_ansi_enabled:
840
+ raise AnalysisException(
841
+ f'[DATATYPE_MISMATCH.BINARY_OP_WRONG_TYPE] Cannot resolve "{spark_function_name}" due to data type mismatch: the binary operator requires the input type ("DOUBLE" or "DECIMAL"), not "STRING".'
842
+ )
843
+ else:
844
+ result_type = DoubleType()
845
+ result_exp = _divnull(
846
+ snowpark_args[0].try_cast(result_type),
847
+ snowpark_args[1].try_cast(result_type),
848
+ )
849
+ case (StringType(), _IntegralType()):
850
+ result_type = DoubleType()
851
+ if spark_sql_ansi_enabled:
852
+ result_exp = _divnull(
853
+ snowpark_args[0].cast(LongType()),
854
+ snowpark_args[1].cast(result_type),
855
+ )
856
+ else:
857
+ result_exp = _divnull(
858
+ snowpark_args[0].try_cast(result_type), snowpark_args[1]
859
+ )
860
+ result_exp = result_exp.cast(result_type)
861
+ case (StringType(), _FractionalType()):
862
+ result_type = DoubleType()
863
+ if spark_sql_ansi_enabled:
864
+ result_exp = _divnull(
865
+ snowpark_args[0].cast(result_type), snowpark_args[1]
866
+ )
867
+ else:
868
+ result_exp = _divnull(
869
+ snowpark_args[0].try_cast(result_type), snowpark_args[1]
870
+ )
871
+ case (_IntegralType(), StringType()):
872
+ result_type = DoubleType()
873
+ if spark_sql_ansi_enabled:
874
+ result_exp = _divnull(
875
+ snowpark_args[0].cast(result_type),
876
+ snowpark_args[1].cast(LongType()),
877
+ )
878
+ else:
879
+ result_exp = _divnull(
880
+ snowpark_args[0], snowpark_args[1].try_cast(result_type)
881
+ )
882
+ result_exp = result_exp.cast(result_type)
883
+ case (_FractionalType(), StringType()):
884
+ result_type = DoubleType()
885
+ if spark_sql_ansi_enabled:
886
+ result_exp = _divnull(
887
+ snowpark_args[0], snowpark_args[1].cast(result_type)
888
+ )
889
+ else:
890
+ result_exp = _divnull(
891
+ snowpark_args[0], snowpark_args[1].try_cast(result_type)
892
+ )
893
+ case (_NumericType(), NullType()) | (NullType(), _NumericType()):
894
+ result_type = DoubleType()
895
+ result_exp = snowpark_fn.lit(None)
896
+ case (_NumericType(), _NumericType()):
897
+ result_type = DoubleType()
898
+ result_exp = _divnull(
899
+ snowpark_args[0].cast(result_type),
900
+ snowpark_args[1].cast(result_type),
901
+ )
902
+ case _:
903
+ raise AnalysisException(
904
+ f'[DATATYPE_MISMATCH.BINARY_OP_DIFF_TYPES] Cannot resolve "{spark_function_name}" due to data type mismatch: the left and right operands of the binary operator have incompatible types ("{snowpark_typed_args[0].typ}" and "{snowpark_typed_args[1].typ}").'
905
+ )
665
906
  case "~":
666
907
  result_exp = TypedColumn(
667
908
  snowpark_fn.bitnot(snowpark_args[0]),
@@ -1765,9 +2006,11 @@ def map_unresolved_function(
1765
2006
  snowpark_args[1], snowpark_args[2], snowpark_args[0]
1766
2007
  )
1767
2008
  else:
2009
+ spark_function_name = f"convert_timezone(current_timezone(), {', '.join(snowpark_arg_names)})"
1768
2010
  result_exp = snowpark_fn.convert_timezone(*snowpark_args)
1769
2011
 
1770
- result_exp = TypedColumn(result_exp, lambda: [TimestampType()])
2012
+ result_type = TimestampType(TimestampTimeZone.NTZ)
2013
+ result_exp = result_exp.cast(result_type)
1771
2014
 
1772
2015
  case "corr":
1773
2016
  col1_type = snowpark_typed_args[0].typ
@@ -2055,11 +2298,8 @@ def map_unresolved_function(
2055
2298
  result_exp = TypedColumn(snowpark_fn.current_date(), lambda: [DateType()])
2056
2299
  spark_function_name = "current_date()"
2057
2300
  case "current_timestamp" | "now":
2058
- result_exp = snowpark_fn.cast(
2059
- snowpark_fn.current_timestamp(),
2060
- get_timestamp_type(),
2061
- )
2062
- result_exp = TypedColumn(result_exp, lambda: [get_timestamp_type()])
2301
+ result_type = TimestampType(TimestampTimeZone.LTZ)
2302
+ result_exp = snowpark_fn.to_timestamp_ltz(snowpark_fn.current_timestamp())
2063
2303
  case "current_timezone":
2064
2304
  result_exp = snowpark_fn.lit(global_config.spark_sql_session_timeZone)
2065
2305
  result_type = StringType()
@@ -2650,7 +2890,7 @@ def map_unresolved_function(
2650
2890
  return results
2651
2891
 
2652
2892
  spark_function_name = f"from_csv({snowpark_arg_names[0]})"
2653
- ddl_schema = parse_ddl_string(snowpark_arg_names[1], True)
2893
+ result_type = map_type_string_to_snowpark_type(snowpark_arg_names[1])
2654
2894
 
2655
2895
  if len(snowpark_arg_names) > 2 and snowpark_arg_names[2].startswith(
2656
2896
  "named_struct"
@@ -2675,8 +2915,7 @@ def map_unresolved_function(
2675
2915
 
2676
2916
  result_exp = snowpark_fn.when(
2677
2917
  snowpark_args[0].is_null(), snowpark_fn.lit(None)
2678
- ).otherwise(snowpark_fn.cast(csv_result, ddl_schema))
2679
- result_type = ddl_schema
2918
+ ).otherwise(snowpark_fn.cast(csv_result, result_type))
2680
2919
  case "from_json":
2681
2920
  # TODO: support options.
2682
2921
  if len(snowpark_args) > 2:
@@ -2903,21 +3142,15 @@ def map_unresolved_function(
2903
3142
  )
2904
3143
  result_type = StringType()
2905
3144
  case "from_utc_timestamp":
2906
-
2907
- @cached_udf(
2908
- input_types=[StringType()],
2909
- return_type=StringType(),
2910
- )
2911
- def map_from_spark_tz(tz):
2912
- return SPARK_TZ_ABBREVIATIONS_OVERRIDES.get(tz, tz)
2913
-
2914
- target_tz = map_from_spark_tz(snowpark_args[1])
3145
+ target_tz = _map_from_spark_tz(snowpark_args[1])
2915
3146
  result_exp = _try_to_cast(
2916
3147
  "try_to_timestamp",
2917
- snowpark_fn.from_utc_timestamp(snowpark_args[0], target_tz),
3148
+ snowpark_fn.from_utc_timestamp(snowpark_args[0], target_tz).cast(
3149
+ TimestampType()
3150
+ ),
2918
3151
  snowpark_args[0],
2919
3152
  )
2920
- result_type = TimestampType(TimestampTimeZone.NTZ)
3153
+ result_type = TimestampType()
2921
3154
  case "get":
2922
3155
  if exp.unresolved_function.arguments[1].HasField("literal"):
2923
3156
  index = unwrap_literal(exp.unresolved_function.arguments[1])
@@ -3823,8 +4056,10 @@ def map_unresolved_function(
3823
4056
  ).otherwise(snowpark_fn.ln(snowpark_args[0]))
3824
4057
  result_type = DoubleType()
3825
4058
  case "localtimestamp":
3826
- result_exp = snowpark_fn.builtin("localtimestamp")()
3827
- result_type = TimestampType(TimestampTimeZone.LTZ)
4059
+ result_type = TimestampType(TimestampTimeZone.NTZ)
4060
+ result_exp = snowpark_fn.to_timestamp_ntz(
4061
+ snowpark_fn.builtin("localtimestamp")()
4062
+ )
3828
4063
  case "locate":
3829
4064
  substr = unwrap_literal(exp.unresolved_function.arguments[0])
3830
4065
  value = snowpark_args[1]
@@ -3999,7 +4234,7 @@ def map_unresolved_function(
3999
4234
  match function_name:
4000
4235
  case "make_timestamp":
4001
4236
  make_function_name = "timestamp_tz_from_parts"
4002
- result_type = TimestampType()
4237
+ result_type = get_timestamp_type()
4003
4238
  case "make_timestamp_ltz":
4004
4239
  make_function_name = "timestamp_ltz_from_parts"
4005
4240
  result_type = TimestampType(TimestampTimeZone.LTZ)
@@ -6552,15 +6787,22 @@ def map_unresolved_function(
6552
6787
  case "timestamp_add":
6553
6788
  # Added to DataFrame functions in 4.0.0 - but can be called from SQL in 3.5.3.
6554
6789
  spark_function_name = f"timestampadd({snowpark_arg_names[0]}, {snowpark_arg_names[1]}, {snowpark_arg_names[2]})"
6790
+
6791
+ typ = snowpark_typed_args[2].typ
6792
+ result_type = (
6793
+ typ
6794
+ if isinstance(typ, TimestampType)
6795
+ else TimestampType(snowpark.types.TimestampTimeZone.LTZ)
6796
+ )
6797
+
6555
6798
  result_exp = snowpark_fn.cast(
6556
6799
  snowpark_fn.dateadd(
6557
6800
  unwrap_literal(exp.unresolved_function.arguments[0]),
6558
6801
  snowpark_args[1],
6559
6802
  snowpark_args[2],
6560
6803
  ),
6561
- TimestampType(snowpark.types.TimestampTimeZone.NTZ),
6804
+ result_type,
6562
6805
  )
6563
- result_type = TimestampType(snowpark.types.TimestampTimeZone.NTZ)
6564
6806
  case "timestamp_diff":
6565
6807
  # Added to DataFrame functions in 4.0.0 - but can be called from SQL in 3.5.3.
6566
6808
  spark_function_name = f"timestampdiff({snowpark_arg_names[0]}, {snowpark_arg_names[1]}, {snowpark_arg_names[2]})"
@@ -6573,9 +6815,9 @@ def map_unresolved_function(
6573
6815
  case "timestamp_micros":
6574
6816
  result_exp = snowpark_fn.cast(
6575
6817
  snowpark_fn.to_timestamp(snowpark_args[0], 6),
6576
- TimestampType(snowpark.types.TimestampTimeZone.NTZ),
6818
+ TimestampType(snowpark.types.TimestampTimeZone.LTZ),
6577
6819
  )
6578
- result_type = TimestampType(snowpark.types.TimestampTimeZone.NTZ)
6820
+ result_type = TimestampType(snowpark.types.TimestampTimeZone.LTZ)
6579
6821
  case "timestamp_millis":
6580
6822
  if not isinstance(snowpark_typed_args[0].typ, _IntegralType):
6581
6823
  raise AnalysisException(
@@ -6583,9 +6825,9 @@ def map_unresolved_function(
6583
6825
  )
6584
6826
  result_exp = snowpark_fn.cast(
6585
6827
  snowpark_fn.to_timestamp(snowpark_args[0] * 1_000, 6),
6586
- TimestampType(snowpark.types.TimestampTimeZone.NTZ),
6828
+ TimestampType(snowpark.types.TimestampTimeZone.LTZ),
6587
6829
  )
6588
- result_type = TimestampType(snowpark.types.TimestampTimeZone.NTZ)
6830
+ result_type = TimestampType(snowpark.types.TimestampTimeZone.LTZ)
6589
6831
  case "timestamp_seconds":
6590
6832
  # Spark allows seconds to be fractional. Snowflake does not allow that
6591
6833
  # even though the documentation explicitly says that it does.
@@ -6598,9 +6840,9 @@ def map_unresolved_function(
6598
6840
  snowpark_fn.to_timestamp(
6599
6841
  snowpark_fn.cast(snowpark_args[0] * 1_000_000, LongType()), 6
6600
6842
  ),
6601
- TimestampType(snowpark.types.TimestampTimeZone.NTZ),
6843
+ TimestampType(snowpark.types.TimestampTimeZone.LTZ),
6602
6844
  )
6603
- result_type = TimestampType(snowpark.types.TimestampTimeZone.NTZ)
6845
+ result_type = TimestampType(snowpark.types.TimestampTimeZone.LTZ)
6604
6846
  case "to_char" | "to_varchar":
6605
6847
  # The structure of the Spark format string must match: [MI|S] [$] [0|9|G|,]* [.|D] [0|9]* [$] [PR|MI|S]
6606
6848
  # Note the grammar above was retrieved from an error message from PySpark, but it is not entirely accurate.
@@ -6746,20 +6988,29 @@ def map_unresolved_function(
6746
6988
  for typed_arg in snowpark_typed_args
6747
6989
  ]
6748
6990
 
6991
+ timezone_conf = global_config.get("spark.sql.session.timeZone")
6992
+
6993
+ # Objects do not preserve keys order in Snowflake, so we need to pass them in the array
6994
+ # Not all the types are preserved in Snowflake Object, timestamps and dates are converted to strings
6995
+ # to properly format them types have to be passed as argument
6749
6996
  @cached_udf(
6750
- input_types=[VariantType(), VariantType()],
6997
+ input_types=[VariantType(), ArrayType(), ArrayType(), VariantType()],
6751
6998
  return_type=StringType(),
6999
+ packages=["jpype1"],
6752
7000
  )
6753
- def _to_csv(col: dict, options: Optional[dict]) -> str:
6754
- sep = ","
7001
+ def _to_csv(
7002
+ col: dict, keys: list, types: list, options: Optional[dict]
7003
+ ) -> str:
7004
+ import datetime
7005
+
7006
+ import jpype
7007
+
6755
7008
  if options is not None:
6756
7009
  if not isinstance(options, dict):
6757
7010
  raise TypeError(
6758
7011
  "[INVALID_OPTIONS.NON_MAP_FUNCTION] Invalid options: Must use the `map()` function for options."
6759
7012
  )
6760
7013
 
6761
- sep = options.get("sep") or sep
6762
-
6763
7014
  python_to_snowflake_type = {
6764
7015
  "str": "STRING",
6765
7016
  "bool": "BOOLEAN",
@@ -6779,22 +7030,166 @@ def map_unresolved_function(
6779
7030
  f'[INVALID_OPTIONS.NON_STRING_TYPE] Invalid options: A type of keys and values in `map()` must be string, but got "MAP<{k_type}, {v_type}>".'
6780
7031
  )
6781
7032
 
7033
+ options = options or {}
7034
+ lowercased_options = {
7035
+ key.lower(): value for key, value in options.items()
7036
+ }
7037
+
7038
+ sep = lowercased_options.get("sep") or (
7039
+ lowercased_options.get("delimiter") or ","
7040
+ )
7041
+ quote = lowercased_options.get("quote") or '"'
7042
+ quote_all = lowercased_options.get("quoteall", "false")
7043
+ escape = lowercased_options.get("escape") or "\\"
7044
+
7045
+ ignore_leading_white_space = lowercased_options.get(
7046
+ "ignoreleadingwhitespace", "true"
7047
+ )
7048
+ ignore_trailing_white_space = lowercased_options.get(
7049
+ "ignoretrailingwhitespace", "true"
7050
+ )
7051
+ null_value = lowercased_options.get("nullvalue") or ""
7052
+ empty_value = lowercased_options.get("emptyvalue") or '""'
7053
+ char_to_escape_quote_escaping = (
7054
+ lowercased_options.get("chartoescapequoteescaping") or escape
7055
+ )
7056
+
7057
+ date_format = lowercased_options.get("dateformat") or "yyyy-MM-dd"
7058
+ timestamp_format = (
7059
+ lowercased_options.get("timestampformat")
7060
+ or "yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]"
7061
+ )
7062
+ timestamp_NTZ_format = (
7063
+ lowercased_options.get("timestampntzformat")
7064
+ or "yyyy-MM-dd'T'HH:mm:ss[.SSS]"
7065
+ )
7066
+
7067
+ def to_boolean(value: str) -> bool:
7068
+ return value.lower() == "true"
7069
+
7070
+ quote_all = to_boolean(quote_all)
7071
+ ignore_leading_white_space = to_boolean(ignore_leading_white_space)
7072
+ ignore_trailing_white_space = to_boolean(ignore_trailing_white_space)
7073
+
7074
+ def escape_str(value: str) -> str:
7075
+ escape_quote = escape + quote if escape != quote else escape
7076
+ return (
7077
+ value.replace(escape, char_to_escape_quote_escaping + escape)
7078
+ .replace(quote, escape_quote)
7079
+ .replace("\r", "\\r")
7080
+ )
7081
+
7082
+ def escape_and_quote_string(value) -> str:
7083
+ if quote_all:
7084
+ return f"{quote}{escape_str(str(value))}{quote}"
7085
+ return str(value)
7086
+
7087
+ time_types = ("date", "timestamp", "timestamp_ntz")
7088
+ maps_timestamps = any(
7089
+ python_type in time_types for python_type in types
7090
+ )
7091
+
7092
+ # Multiple execution of the UDF are done within the same process, that's why we need to check if the JVM was not already started
7093
+ if maps_timestamps and not jpype.isJVMStarted():
7094
+ jpype.startJVM()
7095
+
7096
+ if maps_timestamps:
7097
+ ZonedDateTime = jpype.JClass("java.time.ZonedDateTime")
7098
+ ZoneId = jpype.JClass("java.time.ZoneId")
7099
+ DateTimeFormatter = jpype.JClass(
7100
+ "java.time.format.DateTimeFormatter"
7101
+ )
7102
+ Instant = jpype.JClass("java.time.Instant")
7103
+ LocalDate = jpype.JClass("java.time.LocalDate")
7104
+ LocalDateTime = jpype.JClass("java.time.LocalDateTime")
7105
+ timestamp_formatter = DateTimeFormatter.ofPattern(timestamp_format)
7106
+ timestamp_ntz_formatter = DateTimeFormatter.ofPattern(
7107
+ timestamp_NTZ_format
7108
+ )
7109
+ date_formatter = DateTimeFormatter.ofPattern(date_format)
7110
+
6782
7111
  result = []
6783
- for _, value in col.items():
7112
+ for key, python_type in zip(keys, types):
7113
+ value = col.get(key)
6784
7114
  if value is None:
6785
- result.append("")
7115
+ result.append(escape_and_quote_string(null_value))
7116
+ elif python_type in ("date", "timestamp", "timestamp_ntz"):
7117
+ match python_type:
7118
+ case "date":
7119
+ value = datetime.datetime.strptime(value, "%Y-%m-%d")
7120
+ local_date = LocalDate.of(
7121
+ value.year, value.month, value.day
7122
+ )
7123
+ formatted_date = date_formatter.format(local_date)
7124
+ result.append(escape_and_quote_string(formatted_date))
7125
+ case "timestamp":
7126
+ try:
7127
+ value = datetime.datetime.strptime(
7128
+ value, "%Y-%m-%d %H:%M:%S.%f %z"
7129
+ )
7130
+ except ValueError:
7131
+ # Fallback to the format without microseconds
7132
+ value = datetime.datetime.strptime(
7133
+ value, "%Y-%m-%d %H:%M:%S %z"
7134
+ )
7135
+ instant = Instant.ofEpochMilli(
7136
+ int(value.timestamp() * 1000)
7137
+ )
7138
+ zdt = ZonedDateTime.ofInstant(
7139
+ instant, ZoneId.of(timezone_conf)
7140
+ )
7141
+ str_value = timestamp_formatter.format(zdt)
7142
+ result.append(escape_and_quote_string(str_value))
7143
+ case "timestamp_ntz":
7144
+ try:
7145
+ value = datetime.datetime.strptime(
7146
+ value, "%Y-%m-%d %H:%M:%S.%f"
7147
+ )
7148
+ except ValueError:
7149
+ # Fallback to the format without microseconds
7150
+ value = datetime.datetime.strptime(
7151
+ value, "%Y-%m-%d %H:%M:%S"
7152
+ )
7153
+ timestamp_ntz = LocalDateTime.of(
7154
+ value.year,
7155
+ value.month,
7156
+ value.day,
7157
+ value.hour,
7158
+ value.minute,
7159
+ value.second,
7160
+ value.microsecond * 1000,
7161
+ )
7162
+ str_value = timestamp_ntz_formatter.format(
7163
+ timestamp_ntz
7164
+ )
7165
+ result.append(escape_and_quote_string(str_value))
7166
+ case _:
7167
+ raise ValueError(
7168
+ f"Unable to determine type for value: {python_type}"
7169
+ )
6786
7170
  elif isinstance(value, str):
6787
- if value == "" or any(
6788
- c in value for c in [",", "\n", "\r", '"', "'"]
7171
+ strip_value = (
7172
+ value.lstrip() if ignore_leading_white_space else value
7173
+ )
7174
+ strip_value = (
7175
+ strip_value.rstrip()
7176
+ if ignore_trailing_white_space
7177
+ else strip_value
7178
+ )
7179
+ if strip_value == "":
7180
+ result.append(escape_and_quote_string(empty_value))
7181
+ elif (
7182
+ any(c in value for c in (sep, "\r", "\n", quote))
7183
+ or quote_all
6789
7184
  ):
6790
- value = value.replace("\\", "\\\\").replace('"', '\\"')
6791
- result.append(f'"{value}"')
7185
+ strip_value = escape_str(strip_value)
7186
+ result.append(quote + strip_value + quote)
6792
7187
  else:
6793
- result.append(value)
7188
+ result.append(escape_and_quote_string(strip_value))
6794
7189
  elif isinstance(value, bool):
6795
- result.append(str(value).lower())
7190
+ result.append(escape_and_quote_string(str(value).lower()))
6796
7191
  else:
6797
- result.append(str(value))
7192
+ result.append(escape_and_quote_string(str(value)))
6798
7193
 
6799
7194
  return sep.join(result)
6800
7195
 
@@ -6807,11 +7202,36 @@ def map_unresolved_function(
6807
7202
  "[INVALID_OPTIONS.NON_MAP_FUNCTION] Invalid options: Must use the `map()` function for options."
6808
7203
  )
6809
7204
 
7205
+ def get_snowpark_type_name(snowpark_type: DataType) -> str:
7206
+ return (
7207
+ (
7208
+ "timestamp"
7209
+ if not snowpark_type.tz == snowpark.types.TimestampTimeZone.NTZ
7210
+ else "timestamp_ntz"
7211
+ )
7212
+ if snowpark_type == TimestampType()
7213
+ else snowpark_type.type_name().lower()
7214
+ )
7215
+
7216
+ field_names = snowpark_fn.array_construct(
7217
+ *[
7218
+ snowpark_fn.lit(value)
7219
+ for value in snowpark_typed_args[0].typ.fieldNames
7220
+ ]
7221
+ )
7222
+ field_types = snowpark_fn.array_construct(
7223
+ *[
7224
+ snowpark_fn.lit(get_snowpark_type_name(value.datatype))
7225
+ for value in snowpark_typed_args[0].typ.fields
7226
+ ]
7227
+ )
6810
7228
  match snowpark_args:
6811
7229
  case [csv_data]:
6812
- result_exp = _to_csv(csv_data, snowpark_fn.lit(None))
7230
+ result_exp = _to_csv(
7231
+ csv_data, field_names, field_types, snowpark_fn.lit(None)
7232
+ )
6813
7233
  case [csv_data, options]:
6814
- result_exp = _to_csv(csv_data, options)
7234
+ result_exp = _to_csv(csv_data, field_names, field_types, options)
6815
7235
  case _:
6816
7236
  raise ValueError("Unrecognized from_csv parameters")
6817
7237
  result_type = StringType()
@@ -6912,10 +7332,8 @@ def map_unresolved_function(
6912
7332
  )
6913
7333
  case _:
6914
7334
  raise ValueError(f"Invalid number of arguments to {function_name}")
6915
- result_exp = snowpark_fn.cast(
6916
- result_exp, TimestampType(snowpark.types.TimestampTimeZone.NTZ)
6917
- )
6918
- result_type = TimestampType()
7335
+ result_exp = snowpark_fn.cast(result_exp, get_timestamp_type())
7336
+ result_type = get_timestamp_type()
6919
7337
 
6920
7338
  case "to_timestamp_ltz":
6921
7339
  match (snowpark_typed_args, exp.unresolved_function.arguments):
@@ -6941,7 +7359,12 @@ def map_unresolved_function(
6941
7359
  match (snowpark_typed_args, exp.unresolved_function.arguments):
6942
7360
  case ([e], _):
6943
7361
  result_exp = snowpark_fn.builtin("to_timestamp_ntz")(e.col)
6944
- case ([e, _], _) if type(e.typ) in (DateType, TimestampType):
7362
+ case ([e, _], _) if isinstance(e.typ, DateType):
7363
+ result_exp = snowpark_fn.convert_timezone(
7364
+ snowpark_fn.lit("UTC"),
7365
+ snowpark_fn.builtin("to_timestamp_ntz")(e.col),
7366
+ )
7367
+ case ([e, _], _) if isinstance(e.typ, TimestampType):
6945
7368
  result_exp = snowpark_fn.builtin("to_timestamp_ntz")(e.col)
6946
7369
  case ([e, _], [_, fmt]):
6947
7370
  result_exp = snowpark_fn.builtin("to_timestamp_ntz")(
@@ -7002,25 +7425,17 @@ def map_unresolved_function(
7002
7425
  result_type = LongType()
7003
7426
 
7004
7427
  case "to_utc_timestamp":
7005
-
7006
- @cached_udf(
7007
- input_types=[StringType()],
7008
- return_type=StringType(),
7009
- )
7010
- def map_timezone(short_tz: str) -> str:
7011
- return SPARK_TZ_ABBREVIATIONS_OVERRIDES.get(short_tz, short_tz)
7012
-
7428
+ result_type = TimestampType()
7013
7429
  result_exp = _try_to_cast(
7014
7430
  "try_to_timestamp",
7015
7431
  snowpark_fn.cast(
7016
7432
  snowpark_fn.to_utc_timestamp(
7017
- snowpark_args[0], map_timezone(snowpark_args[1])
7433
+ snowpark_args[0], _map_from_spark_tz(snowpark_args[1])
7018
7434
  ),
7019
- TimestampType(snowpark.types.TimestampTimeZone.NTZ),
7435
+ result_type,
7020
7436
  ),
7021
7437
  snowpark_args[0],
7022
7438
  )
7023
- result_type = TimestampType(snowpark.types.TimestampTimeZone.NTZ)
7024
7439
  case "translate":
7025
7440
  src_alphabet = unwrap_literal(exp.unresolved_function.arguments[1])
7026
7441
  target_alphabet = unwrap_literal(exp.unresolved_function.arguments[2])
@@ -7413,8 +7828,8 @@ def map_unresolved_function(
7413
7828
  )
7414
7829
  case _:
7415
7830
  raise ValueError(f"Invalid number of arguments to {function_name}")
7416
- result_exp = snowpark_fn.cast(result_exp, TimestampType())
7417
- result_type = TimestampType()
7831
+ result_type = get_timestamp_type()
7832
+ result_exp = snowpark_fn.cast(result_exp, result_type)
7418
7833
  case "typeof":
7419
7834
  col_snowpark_typ = snowpark_typed_args[0].typ
7420
7835
  spark_typ = map_snowpark_to_pyspark_types(col_snowpark_typ)
@@ -7684,14 +8099,10 @@ def map_unresolved_function(
7684
8099
  result_exp = result_exp.when(snowpark_args[i], snowpark_args[i + 1])
7685
8100
  result_type_indexes.append(i + 1)
7686
8101
  name_components.append("END")
7687
- result_exp = TypedColumn(
7688
- result_exp,
7689
- lambda: [
7690
- _find_common_type(
7691
- [snowpark_typed_args[i].typ for i in result_type_indexes]
7692
- )
7693
- ],
8102
+ result_type = _find_common_type(
8103
+ [snowpark_typed_args[i].typ for i in result_type_indexes]
7694
8104
  )
8105
+ result_exp = snowpark_fn.cast(result_exp, result_type)
7695
8106
  spark_function_name = " ".join(name_components)
7696
8107
  case "width_bucket":
7697
8108
  width_bucket_fn = snowpark_fn.function("width_bucket")
@@ -7965,6 +8376,18 @@ def map_unresolved_function(
7965
8376
  return spark_col_names, typed_col
7966
8377
 
7967
8378
 
8379
+ def _cast_helper(column: Column, to: DataType) -> Column:
8380
+ if global_config.spark_sql_ansi_enabled:
8381
+ column_mediator = (
8382
+ snowpark_fn.cast(column, StringType())
8383
+ if isinstance(to, DecimalType)
8384
+ else column
8385
+ )
8386
+ return snowpark_fn.cast(column_mediator, to)
8387
+ else:
8388
+ return _try_cast_helper(column, to)
8389
+
8390
+
7968
8391
  def _try_cast_helper(column: Column, to: DataType) -> Column:
7969
8392
  """
7970
8393
  Attempts to cast a given column to a specified data type using the same behaviour as Spark.
@@ -9119,7 +9542,7 @@ def _get_type_precision(typ: DataType) -> tuple[int, int]:
9119
9542
  case LongType():
9120
9543
  return 20, 0 # -9223372036854775808 to 9223372036854775807
9121
9544
  case NullType():
9122
- return 6, 2 # NULL
9545
+ return 0, 0 # NULL
9123
9546
  case _:
9124
9547
  return 38, 0 # Default to maximum precision for other types
9125
9548
 
@@ -9162,58 +9585,70 @@ def _decimal_add_sub_result_type_helper(p1, s1, p2, s2):
9162
9585
  return result_precision, min_scale, return_type_precision, return_type_scale
9163
9586
 
9164
9587
 
9165
- def _decimal_multiply_result_type_helper(p1, s1, p2, s2):
9166
- result_precision = p1 + p2 + 1
9167
- result_scale = s1 + s2
9168
- return_type_precision, return_type_scale = result_precision, result_scale
9169
-
9170
- if result_precision <= 38:
9171
- return result_precision, result_scale, return_type_precision, return_type_scale
9588
+ def _get_decimal_multiplication_result_exp(
9589
+ result_type: DecimalType | DataType,
9590
+ other_type: DataType,
9591
+ snowpark_args: list[Column],
9592
+ ) -> Column:
9593
+ if global_config.spark_sql_ansi_enabled:
9594
+ result_exp = snowpark_args[0] * snowpark_args[1]
9172
9595
  else:
9173
- return_type_precision = 38
9174
-
9175
- min_scale = 6
9176
- while result_scale > min_scale:
9177
- result_scale -= 1
9178
- return_type_scale = result_scale
9179
- result_precision = (p1 - s1) + (p2 - s2) + result_scale + 1
9180
- if result_precision <= 38:
9181
- return (
9182
- result_precision,
9183
- result_scale,
9184
- return_type_precision,
9185
- return_type_scale,
9596
+ if isinstance(other_type, _IntegralType):
9597
+ result_exp = snowpark_args[0].cast(result_type) * snowpark_args[1].cast(
9598
+ result_type
9186
9599
  )
9187
-
9188
- result_precision = (p1 - s1) + (p2 - s2) + min_scale + 1
9189
- return result_precision, min_scale, return_type_precision, return_type_scale
9600
+ else:
9601
+ result_exp = snowpark_args[0].cast(DoubleType()) * snowpark_args[1].cast(
9602
+ DoubleType()
9603
+ )
9604
+ result_exp = _try_cast_helper(result_exp, result_type)
9605
+ return result_exp
9190
9606
 
9191
9607
 
9192
- def _decimal_divide_result_type_helper(p1, s1, p2, s2):
9193
- result_scale = max(6, s1 + p2 + 1)
9194
- result_precision = p1 - s1 + s2 + result_scale
9195
- return_type_precision, return_type_scale = result_precision, result_scale
9196
-
9197
- if result_precision <= 38:
9198
- return result_precision, result_scale, return_type_precision, return_type_scale
9608
+ def _get_decimal_multiplication_result_type(p1, s1, p2, s2) -> DecimalType:
9609
+ result_precision = p1 + p2 + 1
9610
+ result_scale = s1 + s2
9611
+ if result_precision > 38:
9612
+ if result_scale > 6:
9613
+ overflow = result_precision - 38
9614
+ result_scale = max(6, result_scale - overflow)
9615
+ result_precision = 38
9616
+ return DecimalType(result_precision, result_scale)
9617
+
9618
+
9619
+ def _get_decimal_division_result_exp(
9620
+ result_type: DecimalType | DataType,
9621
+ other_type: DataType,
9622
+ overflow_detected: bool,
9623
+ snowpark_args: list[Column],
9624
+ spark_function_name: str,
9625
+ ) -> Column:
9626
+ if isinstance(other_type, DecimalType) and overflow_detected:
9627
+ if global_config.spark_sql_ansi_enabled:
9628
+ raise ArithmeticException(
9629
+ f'[NUMERIC_VALUE_OUT_OF_RANGE] {spark_function_name} cannot be represented as Decimal({result_type.precision}, {result_type.scale}). If necessary set "spark.sql.ansi.enabled" to "false" to bypass this error, and return NULL instead.'
9630
+ )
9631
+ else:
9632
+ result_exp = snowpark_fn.lit(None).cast(result_type)
9199
9633
  else:
9200
- return_type_precision = 38
9634
+ dividend = snowpark_args[0].cast(DoubleType())
9635
+ divisor = snowpark_args[1]
9636
+ result_exp = _divnull(dividend, divisor)
9637
+ result_exp = _cast_helper(result_exp, result_type)
9638
+ return result_exp
9201
9639
 
9202
- min_scale = 6
9203
- while result_scale > min_scale:
9204
- result_scale -= 1
9205
- return_type_scale = result_scale
9206
- result_precision = p1 - s1 + s2 + result_scale
9207
- if result_precision <= 38:
9208
- return (
9209
- result_precision,
9210
- result_scale,
9211
- return_type_precision,
9212
- return_type_scale,
9213
- )
9214
9640
 
9215
- result_precision = p1 - s1 + s2 + min_scale
9216
- return result_precision, min_scale, return_type_precision, return_type_scale
9641
+ def _get_decimal_division_result_type(p1, s1, p2, s2) -> tuple[DecimalType, bool]:
9642
+ overflow_detected = False
9643
+ result_scale = max(6, s1 + p2 + 1)
9644
+ result_precision = p1 - s1 + s2 + result_scale
9645
+ if result_precision > 38:
9646
+ if result_precision > 40:
9647
+ overflow_detected = True
9648
+ overflow = result_precision - 38
9649
+ result_scale = max(6, result_scale - overflow)
9650
+ result_precision = 38
9651
+ return DecimalType(result_precision, result_scale), overflow_detected
9217
9652
 
9218
9653
 
9219
9654
  def _try_arithmetic_helper(
@@ -9408,102 +9843,108 @@ def _try_arithmetic_helper(
9408
9843
  return snowpark_fn.lit(None)
9409
9844
 
9410
9845
 
9411
- def _add_sub_precision_helper(
9412
- typed_args: List[TypedColumn], snowpark_args: List[Column], operation_type: int
9413
- ) -> tuple[Column, tuple[int, int]]:
9414
- p1, s1 = _get_type_precision(typed_args[0].typ)
9415
- p2, s2 = _get_type_precision(typed_args[1].typ)
9416
-
9417
- if isinstance(typed_args[0].typ, _IntegralType) and isinstance(
9418
- typed_args[1].typ, DecimalType
9419
- ):
9420
- new_scale = s2
9421
- new_precision = max(p2, p1 + s2)
9422
- return_type_precision, return_type_scale = new_precision, new_scale
9423
- elif isinstance(typed_args[0].typ, DecimalType) and isinstance(
9424
- typed_args[1].typ, _IntegralType
9425
- ):
9426
- new_scale = s1
9427
- new_precision = max(p1, p2 + s1)
9428
- return_type_precision, return_type_scale = new_precision, new_scale
9429
- else:
9430
- (
9431
- new_precision,
9432
- new_scale,
9433
- return_type_precision,
9434
- return_type_scale,
9435
- ) = _decimal_add_sub_result_type_helper(p1, s1, p2, s2)
9436
-
9437
- if new_precision > 38:
9438
- if global_config.spark_sql_ansi_enabled:
9439
- raise ArithmeticException(
9440
- f'[NUMERIC_VALUE_OUT_OF_RANGE] Precision {new_precision} exceeds maximum allowed precision of 38. If necessary set "spark.sql.ansi.enabled" to "false" to bypass this error, and return NULL instead.'
9846
+ def _get_add_sub_result_type(
9847
+ type1: DataType,
9848
+ type2: DataType,
9849
+ spark_function_name: str,
9850
+ ) -> DataType:
9851
+ result_type = _find_common_type([type1, type2])
9852
+ match result_type:
9853
+ case DecimalType():
9854
+ p1, s1 = _get_type_precision(type1)
9855
+ p2, s2 = _get_type_precision(type2)
9856
+ result_scale = max(s1, s2)
9857
+ result_precision = max(p1 - s1, p2 - s2) + result_scale + 1
9858
+ if result_precision > 38:
9859
+ if result_scale > 6:
9860
+ overflow = result_precision - 38
9861
+ result_scale = max(6, result_scale - overflow)
9862
+ result_precision = 38
9863
+ result_type = DecimalType(result_precision, result_scale)
9864
+ case NullType():
9865
+ result_type = DoubleType()
9866
+ case StringType():
9867
+ match (type1, type2):
9868
+ case (_FractionalType(), _) | (_, _FractionalType()):
9869
+ result_type = DoubleType()
9870
+ case (_IntegralType(), _) | (_, _IntegralType()):
9871
+ result_type = (
9872
+ LongType()
9873
+ if global_config.spark_sql_ansi_enabled
9874
+ else DoubleType()
9875
+ )
9876
+ case _:
9877
+ if global_config.spark_sql_ansi_enabled:
9878
+ raise AnalysisException(
9879
+ f'[DATATYPE_MISMATCH.BINARY_OP_WRONG_TYPE] Cannot resolve "{spark_function_name}" due to data type mismatch: the binary operator requires the input type ("NUMERIC" or "INTERVAL DAY TO SECOND" or "INTERVAL YEAR TO MONTH" or "INTERVAL"), not "STRING".',
9880
+ )
9881
+ else:
9882
+ result_type = DoubleType()
9883
+ case BooleanType():
9884
+ raise AnalysisException(
9885
+ f'[DATATYPE_MISMATCH.BINARY_OP_WRONG_TYPE] Cannot resolve "{spark_function_name}" due to data type mismatch: the binary operator requires the input type ("NUMERIC" or "INTERVAL DAY TO SECOND" or "INTERVAL YEAR TO MONTH" or "INTERVAL"), not "BOOLEAN".',
9441
9886
  )
9442
- return snowpark_fn.lit(None), (return_type_precision, return_type_scale)
9887
+ return result_type
9443
9888
 
9444
- left_operand, right_operand = snowpark_args[0], snowpark_args[1]
9445
9889
 
9446
- result = (
9447
- left_operand + right_operand
9448
- if operation_type == 0
9449
- else left_operand - right_operand
9450
- )
9451
- return snowpark_fn.cast(result, DecimalType(new_precision, new_scale)), (
9452
- return_type_precision,
9453
- return_type_scale,
9454
- )
9890
+ def _get_spark_function_name(
9891
+ col1: TypedColumn,
9892
+ col2: TypedColumn,
9893
+ snowpark_arg_names: list[str],
9894
+ exp: expressions_proto.Expression,
9895
+ default_spark_function_name: str,
9896
+ function_name: str,
9897
+ ):
9898
+ operation_op = function_name
9899
+ match function_name:
9900
+ case "+":
9901
+ operation_func = "date_add"
9902
+ case "-":
9903
+ operation_func = "date_sub"
9904
+ case _:
9905
+ return default_spark_function_name
9906
+ match (col1.typ, col2.typ):
9907
+ case (DateType(), DateType()):
9908
+ date_param_name1 = _get_literal_param_name(exp, 0, snowpark_arg_names[0])
9909
+ date_param_name2 = _get_literal_param_name(exp, 1, snowpark_arg_names[1])
9910
+ return f"({date_param_name1} {operation_op} {date_param_name2})"
9911
+ case (StringType(), DateType()):
9912
+ date_param_name2 = _get_literal_param_name(exp, 1, snowpark_arg_names[1])
9913
+ if "INTERVAL" == col1.col._expr1.pretty_name:
9914
+ return f"{date_param_name2} {operation_op} {snowpark_arg_names[0]}"
9915
+ elif global_config.spark_sql_ansi_enabled and function_name == "+":
9916
+ return f"{operation_func}(cast({date_param_name2} as date), cast({snowpark_arg_names[0]} as double))"
9917
+ else:
9918
+ return f"({snowpark_arg_names[0]} {operation_op} {date_param_name2})"
9919
+ case (DateType(), StringType()):
9920
+ date_param_name1 = _get_literal_param_name(exp, 0, snowpark_arg_names[0])
9921
+ if (
9922
+ global_config.spark_sql_ansi_enabled
9923
+ or "INTERVAL" == col2.col._expr1.pretty_name
9924
+ ):
9925
+ return f"{date_param_name1} {operation_op} {snowpark_arg_names[1]}"
9926
+ else:
9927
+ return f"{operation_func}(cast({date_param_name1} as date), cast({snowpark_arg_names[1]} as double))"
9928
+ case (DateType() as dt, _) | (_, DateType() as dt):
9929
+ date_param_index = 0 if dt == col1.typ else 1
9930
+ date_param_name = _get_literal_param_name(
9931
+ exp, date_param_index, snowpark_arg_names[date_param_index]
9932
+ )
9933
+ return f"{operation_func}({date_param_name}, {snowpark_arg_names[1 - date_param_index]})"
9934
+ case _:
9935
+ return default_spark_function_name
9455
9936
 
9456
9937
 
9457
- def _mul_div_precision_helper(
9458
- typed_args: List[TypedColumn], snowpark_args: List[Column], operation_type: int
9459
- ) -> Column:
9460
- p1, s1 = _get_type_precision(typed_args[0].typ)
9461
- p2, s2 = _get_type_precision(typed_args[1].typ)
9462
- if operation_type == 0: # multiplication
9463
- (
9464
- new_precision,
9465
- new_scale,
9466
- return_type_precision,
9467
- return_type_scale,
9468
- ) = _decimal_multiply_result_type_helper(p1, s1, p2, s2)
9469
- else: # division
9470
- (
9471
- new_precision,
9472
- new_scale,
9473
- return_type_precision,
9474
- return_type_scale,
9475
- ) = _decimal_divide_result_type_helper(p1, s1, p2, s2)
9476
-
9477
- if isinstance(typed_args[0].typ, DecimalType) and isinstance(
9478
- typed_args[1].typ, DecimalType
9479
- ):
9480
- # Overflow check for both decimal types
9481
- if new_precision > 38:
9482
- if global_config.spark_sql_ansi_enabled:
9483
- raise ArithmeticException(
9484
- f'[NUMERIC_VALUE_OUT_OF_RANGE] Precision {new_precision} exceeds maximum allowed precision of 38. If necessary set "spark.sql.ansi.enabled" to "false" to bypass this error, and return NULL instead.'
9485
- )
9486
- return snowpark_fn.lit(None), (return_type_precision, return_type_scale)
9487
-
9488
- # Overflow check if one operand is an Integral
9489
- if new_precision > 38:
9490
- if global_config.spark_sql_ansi_enabled:
9491
- raise ArithmeticException(
9492
- f'[NUMERIC_VALUE_OUT_OF_RANGE] Precision {new_precision} exceeds maximum allowed precision of 38. If necessary set "spark.sql.ansi.enabled" to "false" to bypass this error, and return NULL instead.'
9493
- )
9494
- new_precision = 38
9495
- if new_scale > new_precision:
9496
- new_scale = new_precision
9497
-
9498
- left_operand, right_operand = snowpark_args[0], snowpark_args[1]
9499
- if operation_type == 0: # multiplication
9500
- result = left_operand * right_operand
9501
- else: # division
9502
- result = _divnull(left_operand, right_operand)
9503
- return snowpark_fn.cast(result, DecimalType(new_precision, new_scale)), (
9504
- return_type_precision,
9505
- return_type_scale,
9506
- )
9938
+ def _get_literal_param_name(exp, arg_index: int, default_param_name: str):
9939
+ try:
9940
+ date_param_name = (
9941
+ exp.unresolved_function.arguments[arg_index]
9942
+ .unresolved_function.arguments[0]
9943
+ .literal.string
9944
+ )
9945
+ except (IndexError, AttributeError):
9946
+ date_param_name = default_param_name
9947
+ return date_param_name
9507
9948
 
9508
9949
 
9509
9950
  def _raise_error_helper(return_type: DataType, error_class=None):
@@ -9818,3 +10259,34 @@ def _trim_helper(value: Column, trim_value: Column, trim_type: Column) -> Column
9818
10259
  return value
9819
10260
 
9820
10261
  return _binary_trim_udf(value, trim_value, trim_type)
10262
+
10263
+
10264
+ def _map_from_spark_tz(value: Column) -> Column:
10265
+ return (
10266
+ snowpark_fn.when(value == "ACT", snowpark_fn.lit("Australia/Darwin"))
10267
+ .when(value == "AET", snowpark_fn.lit("Australia/Sydney"))
10268
+ .when(value == "AGT", snowpark_fn.lit("America/Argentina/Buenos_Aires"))
10269
+ .when(value == "ART", snowpark_fn.lit("Africa/Cairo"))
10270
+ .when(value == "AST", snowpark_fn.lit("America/Anchorage"))
10271
+ .when(value == "BET", snowpark_fn.lit("America/Sao_Paulo"))
10272
+ .when(value == "BST", snowpark_fn.lit("Asia/Dhaka"))
10273
+ .when(value == "CAT", snowpark_fn.lit("Africa/Harare"))
10274
+ .when(value == "CNT", snowpark_fn.lit("America/St_Johns"))
10275
+ .when(value == "CST", snowpark_fn.lit("America/Chicago"))
10276
+ .when(value == "CTT", snowpark_fn.lit("Asia/Shanghai"))
10277
+ .when(value == "EAT", snowpark_fn.lit("Africa/Addis_Ababa"))
10278
+ .when(value == "ECT", snowpark_fn.lit("Europe/Paris"))
10279
+ .when(value == "IET", snowpark_fn.lit("America/Indiana/Indianapolis"))
10280
+ .when(value == "IST", snowpark_fn.lit("Asia/Kolkata"))
10281
+ .when(value == "JST", snowpark_fn.lit("Asia/Tokyo"))
10282
+ .when(value == "MIT", snowpark_fn.lit("Pacific/Apia"))
10283
+ .when(value == "NET", snowpark_fn.lit("Asia/Yerevan"))
10284
+ .when(value == "NST", snowpark_fn.lit("Pacific/Auckland"))
10285
+ .when(value == "PLT", snowpark_fn.lit("Asia/Karachi"))
10286
+ .when(value == "PNT", snowpark_fn.lit("America/Phoenix"))
10287
+ .when(value == "PRT", snowpark_fn.lit("America/Puerto_Rico"))
10288
+ .when(value == "PST", snowpark_fn.lit("America/Los_Angeles"))
10289
+ .when(value == "SST", snowpark_fn.lit("Pacific/Guadalcanal"))
10290
+ .when(value == "VST", snowpark_fn.lit("Asia/Ho_Chi_Minh"))
10291
+ .otherwise(value) # Return original timezone if no mapping found
10292
+ )