pyspark-connect 4.0.0.dev2__tar.gz → 4.1.0.dev1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (331) hide show
  1. {pyspark_connect-4.0.0.dev2/pyspark_connect.egg-info → pyspark_connect-4.1.0.dev1}/PKG-INFO +20 -7
  2. {pyspark_connect-4.0.0.dev2/pyspark/logger → pyspark_connect-4.1.0.dev1/pyspark_connect}/__init__.py +4 -5
  3. pyspark_connect-4.1.0.dev1/pyspark_connect/version.py +1 -0
  4. {pyspark_connect-4.0.0.dev2 → pyspark_connect-4.1.0.dev1/pyspark_connect.egg-info}/PKG-INFO +20 -7
  5. pyspark_connect-4.1.0.dev1/pyspark_connect.egg-info/SOURCES.txt +11 -0
  6. pyspark_connect-4.1.0.dev1/pyspark_connect.egg-info/requires.txt +8 -0
  7. pyspark_connect-4.1.0.dev1/pyspark_connect.egg-info/top_level.txt +1 -0
  8. {pyspark_connect-4.0.0.dev2 → pyspark_connect-4.1.0.dev1}/setup.py +34 -109
  9. pyspark_connect-4.0.0.dev2/pyspark/__init__.py +0 -156
  10. pyspark_connect-4.0.0.dev2/pyspark/_globals.py +0 -71
  11. pyspark_connect-4.0.0.dev2/pyspark/_typing.pyi +0 -43
  12. pyspark_connect-4.0.0.dev2/pyspark/accumulators.py +0 -345
  13. pyspark_connect-4.0.0.dev2/pyspark/cloudpickle/__init__.py +0 -18
  14. pyspark_connect-4.0.0.dev2/pyspark/cloudpickle/cloudpickle.py +0 -1487
  15. pyspark_connect-4.0.0.dev2/pyspark/cloudpickle/cloudpickle_fast.py +0 -13
  16. pyspark_connect-4.0.0.dev2/pyspark/cloudpickle/compat.py +0 -18
  17. pyspark_connect-4.0.0.dev2/pyspark/conf.py +0 -282
  18. pyspark_connect-4.0.0.dev2/pyspark/daemon.py +0 -225
  19. pyspark_connect-4.0.0.dev2/pyspark/errors/__init__.py +0 -88
  20. pyspark_connect-4.0.0.dev2/pyspark/errors/error-conditions.json +0 -1171
  21. pyspark_connect-4.0.0.dev2/pyspark/errors/error_classes.py +0 -31
  22. pyspark_connect-4.0.0.dev2/pyspark/errors/exceptions/__init__.py +0 -32
  23. pyspark_connect-4.0.0.dev2/pyspark/errors/exceptions/base.py +0 -418
  24. pyspark_connect-4.0.0.dev2/pyspark/errors/exceptions/captured.py +0 -445
  25. pyspark_connect-4.0.0.dev2/pyspark/errors/exceptions/connect.py +0 -532
  26. pyspark_connect-4.0.0.dev2/pyspark/errors/utils.py +0 -307
  27. pyspark_connect-4.0.0.dev2/pyspark/errors_doc_gen.py +0 -62
  28. pyspark_connect-4.0.0.dev2/pyspark/find_spark_home.py +0 -95
  29. pyspark_connect-4.0.0.dev2/pyspark/install.py +0 -203
  30. pyspark_connect-4.0.0.dev2/pyspark/instrumentation_utils.py +0 -187
  31. pyspark_connect-4.0.0.dev2/pyspark/java_gateway.py +0 -192
  32. pyspark_connect-4.0.0.dev2/pyspark/join.py +0 -119
  33. pyspark_connect-4.0.0.dev2/pyspark/logger/logger.py +0 -196
  34. pyspark_connect-4.0.0.dev2/pyspark/loose_version.py +0 -75
  35. pyspark_connect-4.0.0.dev2/pyspark/ml/__init__.py +0 -71
  36. pyspark_connect-4.0.0.dev2/pyspark/ml/_typing.pyi +0 -84
  37. pyspark_connect-4.0.0.dev2/pyspark/ml/base.py +0 -412
  38. pyspark_connect-4.0.0.dev2/pyspark/ml/classification.py +0 -4346
  39. pyspark_connect-4.0.0.dev2/pyspark/ml/clustering.py +0 -2188
  40. pyspark_connect-4.0.0.dev2/pyspark/ml/common.py +0 -163
  41. pyspark_connect-4.0.0.dev2/pyspark/ml/connect/__init__.py +0 -46
  42. pyspark_connect-4.0.0.dev2/pyspark/ml/connect/base.py +0 -345
  43. pyspark_connect-4.0.0.dev2/pyspark/ml/connect/classification.py +0 -392
  44. pyspark_connect-4.0.0.dev2/pyspark/ml/connect/evaluation.py +0 -291
  45. pyspark_connect-4.0.0.dev2/pyspark/ml/connect/feature.py +0 -411
  46. pyspark_connect-4.0.0.dev2/pyspark/ml/connect/functions.py +0 -33
  47. pyspark_connect-4.0.0.dev2/pyspark/ml/connect/io_utils.py +0 -334
  48. pyspark_connect-4.0.0.dev2/pyspark/ml/connect/pipeline.py +0 -263
  49. pyspark_connect-4.0.0.dev2/pyspark/ml/connect/summarizer.py +0 -121
  50. pyspark_connect-4.0.0.dev2/pyspark/ml/connect/tuning.py +0 -577
  51. pyspark_connect-4.0.0.dev2/pyspark/ml/connect/util.py +0 -175
  52. pyspark_connect-4.0.0.dev2/pyspark/ml/deepspeed/__init__.py +0 -16
  53. pyspark_connect-4.0.0.dev2/pyspark/ml/deepspeed/deepspeed_distributor.py +0 -164
  54. pyspark_connect-4.0.0.dev2/pyspark/ml/dl_util.py +0 -150
  55. pyspark_connect-4.0.0.dev2/pyspark/ml/evaluation.py +0 -1165
  56. pyspark_connect-4.0.0.dev2/pyspark/ml/feature.py +0 -7480
  57. pyspark_connect-4.0.0.dev2/pyspark/ml/fpm.py +0 -543
  58. pyspark_connect-4.0.0.dev2/pyspark/ml/functions.py +0 -868
  59. pyspark_connect-4.0.0.dev2/pyspark/ml/image.py +0 -267
  60. pyspark_connect-4.0.0.dev2/pyspark/ml/linalg/__init__.py +0 -1379
  61. pyspark_connect-4.0.0.dev2/pyspark/ml/model_cache.py +0 -55
  62. pyspark_connect-4.0.0.dev2/pyspark/ml/param/__init__.py +0 -604
  63. pyspark_connect-4.0.0.dev2/pyspark/ml/param/_shared_params_code_gen.py +0 -375
  64. pyspark_connect-4.0.0.dev2/pyspark/ml/param/shared.py +0 -900
  65. pyspark_connect-4.0.0.dev2/pyspark/ml/pipeline.py +0 -457
  66. pyspark_connect-4.0.0.dev2/pyspark/ml/recommendation.py +0 -748
  67. pyspark_connect-4.0.0.dev2/pyspark/ml/regression.py +0 -3333
  68. pyspark_connect-4.0.0.dev2/pyspark/ml/stat.py +0 -530
  69. pyspark_connect-4.0.0.dev2/pyspark/ml/torch/__init__.py +0 -16
  70. pyspark_connect-4.0.0.dev2/pyspark/ml/torch/data.py +0 -102
  71. pyspark_connect-4.0.0.dev2/pyspark/ml/torch/distributor.py +0 -1094
  72. pyspark_connect-4.0.0.dev2/pyspark/ml/torch/log_communication.py +0 -198
  73. pyspark_connect-4.0.0.dev2/pyspark/ml/torch/torch_run_process_wrapper.py +0 -83
  74. pyspark_connect-4.0.0.dev2/pyspark/ml/tree.py +0 -434
  75. pyspark_connect-4.0.0.dev2/pyspark/ml/tuning.py +0 -1750
  76. pyspark_connect-4.0.0.dev2/pyspark/ml/util.py +0 -758
  77. pyspark_connect-4.0.0.dev2/pyspark/ml/wrapper.py +0 -478
  78. pyspark_connect-4.0.0.dev2/pyspark/mllib/__init__.py +0 -44
  79. pyspark_connect-4.0.0.dev2/pyspark/mllib/_typing.pyi +0 -33
  80. pyspark_connect-4.0.0.dev2/pyspark/mllib/classification.py +0 -989
  81. pyspark_connect-4.0.0.dev2/pyspark/mllib/clustering.py +0 -1318
  82. pyspark_connect-4.0.0.dev2/pyspark/mllib/common.py +0 -174
  83. pyspark_connect-4.0.0.dev2/pyspark/mllib/evaluation.py +0 -690
  84. pyspark_connect-4.0.0.dev2/pyspark/mllib/feature.py +0 -1084
  85. pyspark_connect-4.0.0.dev2/pyspark/mllib/fpm.py +0 -233
  86. pyspark_connect-4.0.0.dev2/pyspark/mllib/linalg/__init__.py +0 -1650
  87. pyspark_connect-4.0.0.dev2/pyspark/mllib/linalg/distributed.py +0 -1662
  88. pyspark_connect-4.0.0.dev2/pyspark/mllib/random.py +0 -698
  89. pyspark_connect-4.0.0.dev2/pyspark/mllib/recommendation.py +0 -389
  90. pyspark_connect-4.0.0.dev2/pyspark/mllib/regression.py +0 -1067
  91. pyspark_connect-4.0.0.dev2/pyspark/mllib/stat/KernelDensity.py +0 -59
  92. pyspark_connect-4.0.0.dev2/pyspark/mllib/stat/__init__.py +0 -34
  93. pyspark_connect-4.0.0.dev2/pyspark/mllib/stat/_statistics.py +0 -409
  94. pyspark_connect-4.0.0.dev2/pyspark/mllib/stat/distribution.py +0 -39
  95. pyspark_connect-4.0.0.dev2/pyspark/mllib/stat/test.py +0 -86
  96. pyspark_connect-4.0.0.dev2/pyspark/mllib/tree.py +0 -888
  97. pyspark_connect-4.0.0.dev2/pyspark/mllib/util.py +0 -660
  98. pyspark_connect-4.0.0.dev2/pyspark/pandas/__init__.py +0 -154
  99. pyspark_connect-4.0.0.dev2/pyspark/pandas/_typing.py +0 -52
  100. pyspark_connect-4.0.0.dev2/pyspark/pandas/accessors.py +0 -988
  101. pyspark_connect-4.0.0.dev2/pyspark/pandas/base.py +0 -1715
  102. pyspark_connect-4.0.0.dev2/pyspark/pandas/categorical.py +0 -798
  103. pyspark_connect-4.0.0.dev2/pyspark/pandas/config.py +0 -548
  104. pyspark_connect-4.0.0.dev2/pyspark/pandas/correlation.py +0 -261
  105. pyspark_connect-4.0.0.dev2/pyspark/pandas/data_type_ops/__init__.py +0 -16
  106. pyspark_connect-4.0.0.dev2/pyspark/pandas/data_type_ops/base.py +0 -517
  107. pyspark_connect-4.0.0.dev2/pyspark/pandas/data_type_ops/binary_ops.py +0 -98
  108. pyspark_connect-4.0.0.dev2/pyspark/pandas/data_type_ops/boolean_ops.py +0 -420
  109. pyspark_connect-4.0.0.dev2/pyspark/pandas/data_type_ops/categorical_ops.py +0 -141
  110. pyspark_connect-4.0.0.dev2/pyspark/pandas/data_type_ops/complex_ops.py +0 -145
  111. pyspark_connect-4.0.0.dev2/pyspark/pandas/data_type_ops/date_ops.py +0 -121
  112. pyspark_connect-4.0.0.dev2/pyspark/pandas/data_type_ops/datetime_ops.py +0 -171
  113. pyspark_connect-4.0.0.dev2/pyspark/pandas/data_type_ops/null_ops.py +0 -83
  114. pyspark_connect-4.0.0.dev2/pyspark/pandas/data_type_ops/num_ops.py +0 -579
  115. pyspark_connect-4.0.0.dev2/pyspark/pandas/data_type_ops/string_ops.py +0 -154
  116. pyspark_connect-4.0.0.dev2/pyspark/pandas/data_type_ops/timedelta_ops.py +0 -101
  117. pyspark_connect-4.0.0.dev2/pyspark/pandas/data_type_ops/udt_ops.py +0 -29
  118. pyspark_connect-4.0.0.dev2/pyspark/pandas/datetimes.py +0 -925
  119. pyspark_connect-4.0.0.dev2/pyspark/pandas/exceptions.py +0 -140
  120. pyspark_connect-4.0.0.dev2/pyspark/pandas/extensions.py +0 -388
  121. pyspark_connect-4.0.0.dev2/pyspark/pandas/frame.py +0 -13814
  122. pyspark_connect-4.0.0.dev2/pyspark/pandas/generic.py +0 -3649
  123. pyspark_connect-4.0.0.dev2/pyspark/pandas/groupby.py +0 -4620
  124. pyspark_connect-4.0.0.dev2/pyspark/pandas/indexes/__init__.py +0 -20
  125. pyspark_connect-4.0.0.dev2/pyspark/pandas/indexes/base.py +0 -2676
  126. pyspark_connect-4.0.0.dev2/pyspark/pandas/indexes/category.py +0 -668
  127. pyspark_connect-4.0.0.dev2/pyspark/pandas/indexes/datetimes.py +0 -882
  128. pyspark_connect-4.0.0.dev2/pyspark/pandas/indexes/multi.py +0 -1292
  129. pyspark_connect-4.0.0.dev2/pyspark/pandas/indexes/timedelta.py +0 -205
  130. pyspark_connect-4.0.0.dev2/pyspark/pandas/indexing.py +0 -1864
  131. pyspark_connect-4.0.0.dev2/pyspark/pandas/internal.py +0 -1652
  132. pyspark_connect-4.0.0.dev2/pyspark/pandas/missing/__init__.py +0 -48
  133. pyspark_connect-4.0.0.dev2/pyspark/pandas/missing/common.py +0 -76
  134. pyspark_connect-4.0.0.dev2/pyspark/pandas/missing/frame.py +0 -62
  135. pyspark_connect-4.0.0.dev2/pyspark/pandas/missing/general_functions.py +0 -43
  136. pyspark_connect-4.0.0.dev2/pyspark/pandas/missing/groupby.py +0 -92
  137. pyspark_connect-4.0.0.dev2/pyspark/pandas/missing/indexes.py +0 -184
  138. pyspark_connect-4.0.0.dev2/pyspark/pandas/missing/resample.py +0 -101
  139. pyspark_connect-4.0.0.dev2/pyspark/pandas/missing/scalars.py +0 -29
  140. pyspark_connect-4.0.0.dev2/pyspark/pandas/missing/series.py +0 -68
  141. pyspark_connect-4.0.0.dev2/pyspark/pandas/missing/window.py +0 -168
  142. pyspark_connect-4.0.0.dev2/pyspark/pandas/mlflow.py +0 -236
  143. pyspark_connect-4.0.0.dev2/pyspark/pandas/namespace.py +0 -3852
  144. pyspark_connect-4.0.0.dev2/pyspark/pandas/numpy_compat.py +0 -259
  145. pyspark_connect-4.0.0.dev2/pyspark/pandas/plot/__init__.py +0 -17
  146. pyspark_connect-4.0.0.dev2/pyspark/pandas/plot/core.py +0 -1185
  147. pyspark_connect-4.0.0.dev2/pyspark/pandas/plot/matplotlib.py +0 -972
  148. pyspark_connect-4.0.0.dev2/pyspark/pandas/plot/plotly.py +0 -255
  149. pyspark_connect-4.0.0.dev2/pyspark/pandas/resample.py +0 -791
  150. pyspark_connect-4.0.0.dev2/pyspark/pandas/series.py +0 -7358
  151. pyspark_connect-4.0.0.dev2/pyspark/pandas/spark/__init__.py +0 -16
  152. pyspark_connect-4.0.0.dev2/pyspark/pandas/spark/accessors.py +0 -1274
  153. pyspark_connect-4.0.0.dev2/pyspark/pandas/spark/functions.py +0 -225
  154. pyspark_connect-4.0.0.dev2/pyspark/pandas/spark/utils.py +0 -202
  155. pyspark_connect-4.0.0.dev2/pyspark/pandas/sql_formatter.py +0 -335
  156. pyspark_connect-4.0.0.dev2/pyspark/pandas/sql_processor.py +0 -390
  157. pyspark_connect-4.0.0.dev2/pyspark/pandas/strings.py +0 -2365
  158. pyspark_connect-4.0.0.dev2/pyspark/pandas/supported_api_gen.py +0 -439
  159. pyspark_connect-4.0.0.dev2/pyspark/pandas/testing.py +0 -328
  160. pyspark_connect-4.0.0.dev2/pyspark/pandas/typedef/__init__.py +0 -18
  161. pyspark_connect-4.0.0.dev2/pyspark/pandas/typedef/typehints.py +0 -877
  162. pyspark_connect-4.0.0.dev2/pyspark/pandas/usage_logging/__init__.py +0 -140
  163. pyspark_connect-4.0.0.dev2/pyspark/pandas/usage_logging/usage_logger.py +0 -132
  164. pyspark_connect-4.0.0.dev2/pyspark/pandas/utils.py +0 -1081
  165. pyspark_connect-4.0.0.dev2/pyspark/pandas/window.py +0 -2704
  166. pyspark_connect-4.0.0.dev2/pyspark/profiler.py +0 -526
  167. pyspark_connect-4.0.0.dev2/pyspark/py.typed +0 -1
  168. pyspark_connect-4.0.0.dev2/pyspark/rddsampler.py +0 -115
  169. pyspark_connect-4.0.0.dev2/pyspark/resource/__init__.py +0 -38
  170. pyspark_connect-4.0.0.dev2/pyspark/resource/information.py +0 -69
  171. pyspark_connect-4.0.0.dev2/pyspark/resource/profile.py +0 -342
  172. pyspark_connect-4.0.0.dev2/pyspark/resource/requests.py +0 -550
  173. pyspark_connect-4.0.0.dev2/pyspark/resultiterable.py +0 -45
  174. pyspark_connect-4.0.0.dev2/pyspark/serializers.py +0 -682
  175. pyspark_connect-4.0.0.dev2/pyspark/shell.py +0 -142
  176. pyspark_connect-4.0.0.dev2/pyspark/shuffle.py +0 -854
  177. pyspark_connect-4.0.0.dev2/pyspark/sql/__init__.py +0 -80
  178. pyspark_connect-4.0.0.dev2/pyspark/sql/_typing.pyi +0 -85
  179. pyspark_connect-4.0.0.dev2/pyspark/sql/avro/__init__.py +0 -18
  180. pyspark_connect-4.0.0.dev2/pyspark/sql/avro/functions.py +0 -224
  181. pyspark_connect-4.0.0.dev2/pyspark/sql/catalog.py +0 -1268
  182. pyspark_connect-4.0.0.dev2/pyspark/sql/column.py +0 -1557
  183. pyspark_connect-4.0.0.dev2/pyspark/sql/conf.py +0 -175
  184. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/__init__.py +0 -18
  185. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/_typing.py +0 -92
  186. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/avro/__init__.py +0 -18
  187. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/avro/functions.py +0 -139
  188. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/catalog.py +0 -365
  189. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/client/__init__.py +0 -23
  190. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/client/artifact.py +0 -444
  191. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/client/core.py +0 -1920
  192. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/client/logging.py +0 -61
  193. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/client/reattach.py +0 -334
  194. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/client/retries.py +0 -317
  195. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/column.py +0 -600
  196. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/conf.py +0 -153
  197. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/conversion.py +0 -584
  198. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/dataframe.py +0 -2344
  199. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/datasource.py +0 -47
  200. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/expressions.py +0 -1142
  201. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/functions/__init__.py +0 -21
  202. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/functions/builtin.py +0 -4203
  203. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/functions/partitioning.py +0 -112
  204. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/group.py +0 -496
  205. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/merge.py +0 -253
  206. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/observation.py +0 -113
  207. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/plan.py +0 -2503
  208. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/profiler.py +0 -41
  209. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/__init__.py +0 -25
  210. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/base_pb2.py +0 -230
  211. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/base_pb2.pyi +0 -3736
  212. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/base_pb2_grpc.py +0 -517
  213. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/catalog_pb2.py +0 -110
  214. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/catalog_pb2.pyi +0 -1133
  215. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/commands_pb2.py +0 -174
  216. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/commands_pb2.pyi +0 -2306
  217. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/common_pb2.py +0 -66
  218. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/common_pb2.pyi +0 -349
  219. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/example_plugins_pb2.py +0 -54
  220. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/example_plugins_pb2.pyi +0 -112
  221. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/expressions_pb2.py +0 -123
  222. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/expressions_pb2.pyi +0 -1803
  223. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/relations_pb2.py +0 -223
  224. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/relations_pb2.pyi +0 -4072
  225. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/types_pb2.py +0 -98
  226. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/types_pb2.pyi +0 -1012
  227. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/protobuf/__init__.py +0 -18
  228. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/protobuf/functions.py +0 -165
  229. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/readwriter.py +0 -1015
  230. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/resource/__init__.py +0 -16
  231. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/resource/profile.py +0 -69
  232. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/session.py +0 -1137
  233. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/shell/__init__.py +0 -26
  234. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/shell/progress.py +0 -212
  235. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/sql_formatter.py +0 -76
  236. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/streaming/__init__.py +0 -16
  237. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/streaming/query.py +0 -464
  238. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/streaming/readwriter.py +0 -720
  239. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/streaming/worker/__init__.py +0 -18
  240. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/streaming/worker/foreach_batch_worker.py +0 -101
  241. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/streaming/worker/listener_worker.py +0 -115
  242. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/types.py +0 -380
  243. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/udf.py +0 -319
  244. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/udtf.py +0 -217
  245. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/utils.py +0 -100
  246. pyspark_connect-4.0.0.dev2/pyspark/sql/connect/window.py +0 -189
  247. pyspark_connect-4.0.0.dev2/pyspark/sql/context.py +0 -816
  248. pyspark_connect-4.0.0.dev2/pyspark/sql/dataframe.py +0 -6550
  249. pyspark_connect-4.0.0.dev2/pyspark/sql/datasource.py +0 -822
  250. pyspark_connect-4.0.0.dev2/pyspark/sql/datasource_internal.py +0 -146
  251. pyspark_connect-4.0.0.dev2/pyspark/sql/functions/__init__.py +0 -21
  252. pyspark_connect-4.0.0.dev2/pyspark/sql/functions/builtin.py +0 -21929
  253. pyspark_connect-4.0.0.dev2/pyspark/sql/functions/partitioning.py +0 -252
  254. pyspark_connect-4.0.0.dev2/pyspark/sql/group.py +0 -551
  255. pyspark_connect-4.0.0.dev2/pyspark/sql/merge.py +0 -253
  256. pyspark_connect-4.0.0.dev2/pyspark/sql/metrics.py +0 -287
  257. pyspark_connect-4.0.0.dev2/pyspark/sql/observation.py +0 -170
  258. pyspark_connect-4.0.0.dev2/pyspark/sql/pandas/__init__.py +0 -21
  259. pyspark_connect-4.0.0.dev2/pyspark/sql/pandas/_typing/__init__.pyi +0 -356
  260. pyspark_connect-4.0.0.dev2/pyspark/sql/pandas/_typing/protocols/__init__.pyi +0 -17
  261. pyspark_connect-4.0.0.dev2/pyspark/sql/pandas/_typing/protocols/frame.pyi +0 -20
  262. pyspark_connect-4.0.0.dev2/pyspark/sql/pandas/_typing/protocols/series.pyi +0 -20
  263. pyspark_connect-4.0.0.dev2/pyspark/sql/pandas/conversion.py +0 -838
  264. pyspark_connect-4.0.0.dev2/pyspark/sql/pandas/functions.py +0 -545
  265. pyspark_connect-4.0.0.dev2/pyspark/sql/pandas/functions.pyi +0 -132
  266. pyspark_connect-4.0.0.dev2/pyspark/sql/pandas/group_ops.py +0 -910
  267. pyspark_connect-4.0.0.dev2/pyspark/sql/pandas/map_ops.py +0 -127
  268. pyspark_connect-4.0.0.dev2/pyspark/sql/pandas/serializers.py +0 -1192
  269. pyspark_connect-4.0.0.dev2/pyspark/sql/pandas/typehints.py +0 -178
  270. pyspark_connect-4.0.0.dev2/pyspark/sql/pandas/types.py +0 -1346
  271. pyspark_connect-4.0.0.dev2/pyspark/sql/pandas/utils.py +0 -96
  272. pyspark_connect-4.0.0.dev2/pyspark/sql/profiler.py +0 -505
  273. pyspark_connect-4.0.0.dev2/pyspark/sql/protobuf/__init__.py +0 -18
  274. pyspark_connect-4.0.0.dev2/pyspark/sql/protobuf/functions.py +0 -336
  275. pyspark_connect-4.0.0.dev2/pyspark/sql/readwriter.py +0 -2544
  276. pyspark_connect-4.0.0.dev2/pyspark/sql/session.py +0 -2340
  277. pyspark_connect-4.0.0.dev2/pyspark/sql/sql_formatter.py +0 -86
  278. pyspark_connect-4.0.0.dev2/pyspark/sql/streaming/StateMessage_pb2.py +0 -73
  279. pyspark_connect-4.0.0.dev2/pyspark/sql/streaming/StateMessage_pb2.pyi +0 -179
  280. pyspark_connect-4.0.0.dev2/pyspark/sql/streaming/__init__.py +0 -25
  281. pyspark_connect-4.0.0.dev2/pyspark/sql/streaming/listener.py +0 -1112
  282. pyspark_connect-4.0.0.dev2/pyspark/sql/streaming/python_streaming_source_runner.py +0 -217
  283. pyspark_connect-4.0.0.dev2/pyspark/sql/streaming/query.py +0 -751
  284. pyspark_connect-4.0.0.dev2/pyspark/sql/streaming/readwriter.py +0 -1826
  285. pyspark_connect-4.0.0.dev2/pyspark/sql/streaming/state.py +0 -297
  286. pyspark_connect-4.0.0.dev2/pyspark/sql/streaming/stateful_processor.py +0 -168
  287. pyspark_connect-4.0.0.dev2/pyspark/sql/streaming/stateful_processor_api_client.py +0 -170
  288. pyspark_connect-4.0.0.dev2/pyspark/sql/streaming/value_state_client.py +0 -105
  289. pyspark_connect-4.0.0.dev2/pyspark/sql/types.py +0 -3332
  290. pyspark_connect-4.0.0.dev2/pyspark/sql/udf.py +0 -786
  291. pyspark_connect-4.0.0.dev2/pyspark/sql/udtf.py +0 -505
  292. pyspark_connect-4.0.0.dev2/pyspark/sql/utils.py +0 -434
  293. pyspark_connect-4.0.0.dev2/pyspark/sql/variant_utils.py +0 -726
  294. pyspark_connect-4.0.0.dev2/pyspark/sql/window.py +0 -429
  295. pyspark_connect-4.0.0.dev2/pyspark/sql/worker/__init__.py +0 -16
  296. pyspark_connect-4.0.0.dev2/pyspark/sql/worker/analyze_udtf.py +0 -270
  297. pyspark_connect-4.0.0.dev2/pyspark/sql/worker/commit_data_source_write.py +0 -115
  298. pyspark_connect-4.0.0.dev2/pyspark/sql/worker/create_data_source.py +0 -192
  299. pyspark_connect-4.0.0.dev2/pyspark/sql/worker/lookup_data_sources.py +0 -100
  300. pyspark_connect-4.0.0.dev2/pyspark/sql/worker/plan_data_source_read.py +0 -373
  301. pyspark_connect-4.0.0.dev2/pyspark/sql/worker/python_streaming_sink_runner.py +0 -151
  302. pyspark_connect-4.0.0.dev2/pyspark/sql/worker/write_into_data_source.py +0 -247
  303. pyspark_connect-4.0.0.dev2/pyspark/statcounter.py +0 -165
  304. pyspark_connect-4.0.0.dev2/pyspark/storagelevel.py +0 -97
  305. pyspark_connect-4.0.0.dev2/pyspark/streaming/__init__.py +0 -22
  306. pyspark_connect-4.0.0.dev2/pyspark/streaming/context.py +0 -471
  307. pyspark_connect-4.0.0.dev2/pyspark/streaming/dstream.py +0 -931
  308. pyspark_connect-4.0.0.dev2/pyspark/streaming/kinesis.py +0 -205
  309. pyspark_connect-4.0.0.dev2/pyspark/streaming/listener.py +0 -83
  310. pyspark_connect-4.0.0.dev2/pyspark/streaming/util.py +0 -168
  311. pyspark_connect-4.0.0.dev2/pyspark/taskcontext.py +0 -506
  312. pyspark_connect-4.0.0.dev2/pyspark/testing/__init__.py +0 -19
  313. pyspark_connect-4.0.0.dev2/pyspark/testing/connectutils.py +0 -222
  314. pyspark_connect-4.0.0.dev2/pyspark/testing/mllibutils.py +0 -30
  315. pyspark_connect-4.0.0.dev2/pyspark/testing/mlutils.py +0 -279
  316. pyspark_connect-4.0.0.dev2/pyspark/testing/pandasutils.py +0 -645
  317. pyspark_connect-4.0.0.dev2/pyspark/testing/sqlutils.py +0 -295
  318. pyspark_connect-4.0.0.dev2/pyspark/testing/streamingutils.py +0 -177
  319. pyspark_connect-4.0.0.dev2/pyspark/testing/utils.py +0 -1016
  320. pyspark_connect-4.0.0.dev2/pyspark/traceback_utils.py +0 -81
  321. pyspark_connect-4.0.0.dev2/pyspark/util.py +0 -801
  322. pyspark_connect-4.0.0.dev2/pyspark/version.py +0 -1
  323. pyspark_connect-4.0.0.dev2/pyspark/worker.py +0 -1938
  324. pyspark_connect-4.0.0.dev2/pyspark/worker_util.py +0 -189
  325. pyspark_connect-4.0.0.dev2/pyspark_connect.egg-info/SOURCES.txt +0 -326
  326. pyspark_connect-4.0.0.dev2/pyspark_connect.egg-info/requires.txt +0 -6
  327. pyspark_connect-4.0.0.dev2/pyspark_connect.egg-info/top_level.txt +0 -1
  328. {pyspark_connect-4.0.0.dev2 → pyspark_connect-4.1.0.dev1}/MANIFEST.in +0 -0
  329. {pyspark_connect-4.0.0.dev2 → pyspark_connect-4.1.0.dev1}/README.md +0 -0
  330. {pyspark_connect-4.0.0.dev2 → pyspark_connect-4.1.0.dev1}/pyspark_connect.egg-info/dependency_links.txt +0 -0
  331. {pyspark_connect-4.0.0.dev2 → pyspark_connect-4.1.0.dev1}/setup.cfg +0 -0
@@ -1,7 +1,7 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.4
2
2
  Name: pyspark-connect
3
- Version: 4.0.0.dev2
4
- Summary: Python Spark Connect client for Apache Spark
3
+ Version: 4.1.0.dev1
4
+ Summary: Apache Spark Python API with Spark Connect by default
5
5
  Home-page: https://github.com/apache/spark/tree/master/python
6
6
  Author: Spark Developers
7
7
  Author-email: dev@spark.apache.org
@@ -12,17 +12,30 @@ Classifier: Programming Language :: Python :: 3.9
12
12
  Classifier: Programming Language :: Python :: 3.10
13
13
  Classifier: Programming Language :: Python :: 3.11
14
14
  Classifier: Programming Language :: Python :: 3.12
15
+ Classifier: Programming Language :: Python :: 3.13
15
16
  Classifier: Programming Language :: Python :: Implementation :: CPython
16
17
  Classifier: Programming Language :: Python :: Implementation :: PyPy
17
18
  Classifier: Typing :: Typed
18
19
  Requires-Python: >=3.9
19
20
  Description-Content-Type: text/markdown
21
+ Requires-Dist: pyspark==4.1.0.dev1
20
22
  Requires-Dist: pandas>=2.0.0
21
- Requires-Dist: pyarrow>=10.0.0
22
- Requires-Dist: grpcio>=1.59.3
23
- Requires-Dist: grpcio-status>=1.59.3
24
- Requires-Dist: googleapis-common-protos>=1.56.4
23
+ Requires-Dist: pyarrow>=11.0.0
24
+ Requires-Dist: grpcio>=1.67.0
25
+ Requires-Dist: grpcio-status>=1.67.0
26
+ Requires-Dist: googleapis-common-protos>=1.65.0
25
27
  Requires-Dist: numpy>=1.21
28
+ Requires-Dist: pyyaml>=3.11
29
+ Dynamic: author
30
+ Dynamic: author-email
31
+ Dynamic: classifier
32
+ Dynamic: description
33
+ Dynamic: description-content-type
34
+ Dynamic: home-page
35
+ Dynamic: license
36
+ Dynamic: requires-dist
37
+ Dynamic: requires-python
38
+ Dynamic: summary
26
39
 
27
40
  # Apache Spark
28
41
 
@@ -16,12 +16,11 @@
16
16
  #
17
17
 
18
18
  """
19
- PySpark logging
19
+ A marker package to make Spark Connect default.
20
20
  """
21
- from pyspark.logger.logger import ( # noqa: F401
22
- PySparkLogger,
23
- )
21
+
22
+ from pyspark_connect.version import __version__
24
23
 
25
24
  __all__ = [
26
- "PySparkLogger",
25
+ "__version__",
27
26
  ]
@@ -0,0 +1 @@
1
+ __version__: str = '4.1.0.dev1'
@@ -1,7 +1,7 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.4
2
2
  Name: pyspark-connect
3
- Version: 4.0.0.dev2
4
- Summary: Python Spark Connect client for Apache Spark
3
+ Version: 4.1.0.dev1
4
+ Summary: Apache Spark Python API with Spark Connect by default
5
5
  Home-page: https://github.com/apache/spark/tree/master/python
6
6
  Author: Spark Developers
7
7
  Author-email: dev@spark.apache.org
@@ -12,17 +12,30 @@ Classifier: Programming Language :: Python :: 3.9
12
12
  Classifier: Programming Language :: Python :: 3.10
13
13
  Classifier: Programming Language :: Python :: 3.11
14
14
  Classifier: Programming Language :: Python :: 3.12
15
+ Classifier: Programming Language :: Python :: 3.13
15
16
  Classifier: Programming Language :: Python :: Implementation :: CPython
16
17
  Classifier: Programming Language :: Python :: Implementation :: PyPy
17
18
  Classifier: Typing :: Typed
18
19
  Requires-Python: >=3.9
19
20
  Description-Content-Type: text/markdown
21
+ Requires-Dist: pyspark==4.1.0.dev1
20
22
  Requires-Dist: pandas>=2.0.0
21
- Requires-Dist: pyarrow>=10.0.0
22
- Requires-Dist: grpcio>=1.59.3
23
- Requires-Dist: grpcio-status>=1.59.3
24
- Requires-Dist: googleapis-common-protos>=1.56.4
23
+ Requires-Dist: pyarrow>=11.0.0
24
+ Requires-Dist: grpcio>=1.67.0
25
+ Requires-Dist: grpcio-status>=1.67.0
26
+ Requires-Dist: googleapis-common-protos>=1.65.0
25
27
  Requires-Dist: numpy>=1.21
28
+ Requires-Dist: pyyaml>=3.11
29
+ Dynamic: author
30
+ Dynamic: author-email
31
+ Dynamic: classifier
32
+ Dynamic: description
33
+ Dynamic: description-content-type
34
+ Dynamic: home-page
35
+ Dynamic: license
36
+ Dynamic: requires-dist
37
+ Dynamic: requires-python
38
+ Dynamic: summary
26
39
 
27
40
  # Apache Spark
28
41
 
@@ -0,0 +1,11 @@
1
+ MANIFEST.in
2
+ README.md
3
+ setup.cfg
4
+ setup.py
5
+ pyspark_connect/__init__.py
6
+ pyspark_connect/version.py
7
+ pyspark_connect.egg-info/PKG-INFO
8
+ pyspark_connect.egg-info/SOURCES.txt
9
+ pyspark_connect.egg-info/dependency_links.txt
10
+ pyspark_connect.egg-info/requires.txt
11
+ pyspark_connect.egg-info/top_level.txt
@@ -0,0 +1,8 @@
1
+ pyspark==4.1.0.dev1
2
+ pandas>=2.0.0
3
+ pyarrow>=11.0.0
4
+ grpcio>=1.67.0
5
+ grpcio-status>=1.67.0
6
+ googleapis-common-protos>=1.65.0
7
+ numpy>=1.21
8
+ pyyaml>=3.11
@@ -0,0 +1 @@
1
+ pyspark_connect
@@ -17,20 +17,20 @@
17
17
  # limitations under the License.
18
18
 
19
19
  # cd python
20
- # python packaging/classic/setup.py sdist
20
+ # python packaging/connect/setup.py sdist
21
21
 
22
- # cd python/packaging/classic
22
+ # cd python/packaging/connect
23
23
  # python setup.py sdist
24
24
 
25
25
  import sys
26
26
  from setuptools import setup
27
27
  import os
28
- from shutil import copyfile, move
28
+ from shutil import copyfile, copytree, move, rmtree
29
29
  import glob
30
30
  from pathlib import Path
31
31
 
32
32
  if (
33
- # When we package, the parent diectory 'connect' dir
33
+ # When we package, the parent directory 'connect' dir
34
34
  # (as we pip install -e python/packaging/connect)
35
35
  os.getcwd() == str(Path(__file__).parent.absolute())
36
36
  and str(Path(__file__).parent.name) == "connect"
@@ -47,16 +47,6 @@ if (
47
47
  # Move to spark/python
48
48
  os.chdir(Path(__file__).parent.parent.parent.absolute())
49
49
 
50
- try:
51
- exec(open("pyspark/version.py").read())
52
- except IOError:
53
- print(
54
- "Failed to load PySpark version file for packaging. You must be in Spark's python dir.",
55
- file=sys.stderr,
56
- )
57
- sys.exit(-1)
58
- VERSION = __version__ # noqa
59
-
60
50
  # Check and see if we are under the spark path in which case we need to build the symlink farm.
61
51
  # This is important because we only want to build the symlink farm while under Spark otherwise we
62
52
  # want to use the symlink farm. And if the symlink farm exists under while under Spark (e.g. a
@@ -65,142 +55,74 @@ in_spark = os.path.isfile("../core/src/main/scala/org/apache/spark/SparkContext.
65
55
  os.path.isfile("../RELEASE") and len(glob.glob("../jars/spark*core*.jar")) == 1
66
56
  )
67
57
 
68
- test_packages = []
69
- if "SPARK_TESTING" in os.environ:
70
- test_packages = [
71
- "pyspark.tests", # for Memory profiler parity tests
72
- "pyspark.testing",
73
- "pyspark.resource.tests",
74
- "pyspark.sql.tests",
75
- "pyspark.sql.tests.connect",
76
- "pyspark.sql.tests.connect.streaming",
77
- "pyspark.sql.tests.connect.client",
78
- "pyspark.sql.tests.connect.shell",
79
- "pyspark.sql.tests.pandas",
80
- "pyspark.sql.tests.streaming",
81
- "pyspark.ml.tests.connect",
82
- "pyspark.pandas.tests",
83
- "pyspark.pandas.tests.computation",
84
- "pyspark.pandas.tests.data_type_ops",
85
- "pyspark.pandas.tests.diff_frames_ops",
86
- "pyspark.pandas.tests.frame",
87
- "pyspark.pandas.tests.groupby",
88
- "pyspark.pandas.tests.indexes",
89
- "pyspark.pandas.tests.io",
90
- "pyspark.pandas.tests.plot",
91
- "pyspark.pandas.tests.resample",
92
- "pyspark.pandas.tests.reshape",
93
- "pyspark.pandas.tests.series",
94
- "pyspark.pandas.tests.window",
95
- "pyspark.pandas.tests.connect",
96
- "pyspark.pandas.tests.connect.computation",
97
- "pyspark.pandas.tests.connect.data_type_ops",
98
- "pyspark.pandas.tests.connect.diff_frames_ops",
99
- "pyspark.pandas.tests.connect.frame",
100
- "pyspark.pandas.tests.connect.groupby",
101
- "pyspark.pandas.tests.connect.indexes",
102
- "pyspark.pandas.tests.connect.io",
103
- "pyspark.pandas.tests.connect.plot",
104
- "pyspark.pandas.tests.connect.resample",
105
- "pyspark.pandas.tests.connect.reshape",
106
- "pyspark.pandas.tests.connect.series",
107
- "pyspark.pandas.tests.connect.window",
108
- "pyspark.logger.tests",
109
- "pyspark.logger.tests.connect",
110
- ]
111
-
112
58
  try:
113
59
  if in_spark:
114
60
  # !!HACK ALTERT!!
115
61
  # 1. `setup.py` has to be located with the same directory with the package.
116
62
  # Therefore, we copy the current file, and place it at `spark/python` directory.
117
63
  # After that, we remove it in the end.
118
- # 2. Here it renames `lib` to `lib.ack` so MANIFEST.in does not pick `py4j` up.
119
- # We rename it back in the end.
64
+ # 2. Here it renames `pyspark` and `lib` to `pyspark.back` and `lib.back` so MANIFEST.in
65
+ # does not pick `pyspark` and `py4j` up. We rename it back in the end.
66
+ move("pyspark", "pyspark.back")
120
67
  move("lib", "lib.back")
121
68
  copyfile("packaging/connect/setup.py", "setup.py")
122
69
  copyfile("packaging/connect/setup.cfg", "setup.cfg")
70
+ copytree("packaging/connect/pyspark_connect", "pyspark_connect")
71
+ copyfile("pyspark.back/version.py", "pyspark_connect/version.py")
72
+
73
+ try:
74
+ exec(open("pyspark_connect/version.py").read())
75
+ except IOError:
76
+ print(
77
+ "Failed to load PySpark Connect version file for packaging. "
78
+ "You must be in Spark's python dir.",
79
+ file=sys.stderr,
80
+ )
81
+ sys.exit(-1)
82
+ VERSION = __version__ # noqa
123
83
 
124
84
  # If you are changing the versions here, please also change ./python/pyspark/sql/pandas/utils.py
125
85
  # For Arrow, you should also check ./pom.xml and ensure there are no breaking changes in the
126
86
  # binary format protocol with the Java version, see ARROW_HOME/format/* for specifications.
127
- # Also don't forget to update python/docs/source/getting_started/install.rst, and
128
- # python/packaging/classic/setup.py
87
+ # Also don't forget to update python/docs/source/getting_started/install.rst,
88
+ # python/packaging/classic/setup.py, and python/packaging/client/setup.py
129
89
  _minimum_pandas_version = "2.0.0"
130
90
  _minimum_numpy_version = "1.21"
131
- _minimum_pyarrow_version = "10.0.0"
132
- _minimum_grpc_version = "1.59.3"
133
- _minimum_googleapis_common_protos_version = "1.56.4"
91
+ _minimum_pyarrow_version = "11.0.0"
92
+ _minimum_grpc_version = "1.67.0"
93
+ _minimum_googleapis_common_protos_version = "1.65.0"
94
+ _minimum_pyyaml_version = "3.11"
134
95
 
135
96
  with open("README.md") as f:
136
97
  long_description = f.read()
137
98
 
138
99
  connect_packages = [
139
- "pyspark",
140
- "pyspark.cloudpickle",
141
- "pyspark.mllib",
142
- "pyspark.mllib.linalg",
143
- "pyspark.mllib.stat",
144
- "pyspark.ml",
145
- "pyspark.ml.connect",
146
- "pyspark.ml.linalg",
147
- "pyspark.ml.param",
148
- "pyspark.ml.torch",
149
- "pyspark.ml.deepspeed",
150
- "pyspark.sql",
151
- "pyspark.sql.avro",
152
- "pyspark.sql.connect",
153
- "pyspark.sql.connect.avro",
154
- "pyspark.sql.connect.client",
155
- "pyspark.sql.connect.functions",
156
- "pyspark.sql.connect.proto",
157
- "pyspark.sql.connect.protobuf",
158
- "pyspark.sql.connect.resource",
159
- "pyspark.sql.connect.shell",
160
- "pyspark.sql.connect.streaming",
161
- "pyspark.sql.connect.streaming.worker",
162
- "pyspark.sql.functions",
163
- "pyspark.sql.pandas",
164
- "pyspark.sql.protobuf",
165
- "pyspark.sql.streaming",
166
- "pyspark.sql.worker",
167
- "pyspark.streaming",
168
- "pyspark.pandas",
169
- "pyspark.pandas.data_type_ops",
170
- "pyspark.pandas.indexes",
171
- "pyspark.pandas.missing",
172
- "pyspark.pandas.plot",
173
- "pyspark.pandas.spark",
174
- "pyspark.pandas.typedef",
175
- "pyspark.pandas.usage_logging",
176
- "pyspark.testing",
177
- "pyspark.resource",
178
- "pyspark.errors",
179
- "pyspark.errors.exceptions",
180
- "pyspark.logger",
100
+ "pyspark_connect",
181
101
  ]
182
102
 
183
103
  setup(
184
104
  name="pyspark-connect",
185
105
  version=VERSION,
186
- description="Python Spark Connect client for Apache Spark",
106
+ description="Apache Spark Python API with Spark Connect by default",
187
107
  long_description=long_description,
188
108
  long_description_content_type="text/markdown",
189
109
  author="Spark Developers",
190
110
  author_email="dev@spark.apache.org",
191
111
  url="https://github.com/apache/spark/tree/master/python",
192
- packages=connect_packages + test_packages,
112
+ packages=connect_packages,
193
113
  include_package_data=True,
194
114
  license="http://www.apache.org/licenses/LICENSE-2.0",
195
115
  # Don't forget to update python/docs/source/getting_started/install.rst
196
116
  # if you're updating the versions or dependencies.
197
117
  install_requires=[
118
+ "pyspark==%s" % VERSION,
198
119
  "pandas>=%s" % _minimum_pandas_version,
199
120
  "pyarrow>=%s" % _minimum_pyarrow_version,
200
121
  "grpcio>=%s" % _minimum_grpc_version,
201
122
  "grpcio-status>=%s" % _minimum_grpc_version,
202
123
  "googleapis-common-protos>=%s" % _minimum_googleapis_common_protos_version,
203
124
  "numpy>=%s" % _minimum_numpy_version,
125
+ "pyyaml>=%s" % _minimum_pyyaml_version,
204
126
  ],
205
127
  python_requires=">=3.9",
206
128
  classifiers=[
@@ -210,6 +132,7 @@ try:
210
132
  "Programming Language :: Python :: 3.10",
211
133
  "Programming Language :: Python :: 3.11",
212
134
  "Programming Language :: Python :: 3.12",
135
+ "Programming Language :: Python :: 3.13",
213
136
  "Programming Language :: Python :: Implementation :: CPython",
214
137
  "Programming Language :: Python :: Implementation :: PyPy",
215
138
  "Typing :: Typed",
@@ -217,6 +140,8 @@ try:
217
140
  )
218
141
  finally:
219
142
  if in_spark:
143
+ move("pyspark.back", "pyspark")
220
144
  move("lib.back", "lib")
221
145
  os.remove("setup.py")
222
146
  os.remove("setup.cfg")
147
+ rmtree("pyspark_connect")
@@ -1,156 +0,0 @@
1
- #
2
- # Licensed to the Apache Software Foundation (ASF) under one or more
3
- # contributor license agreements. See the NOTICE file distributed with
4
- # this work for additional information regarding copyright ownership.
5
- # The ASF licenses this file to You under the Apache License, Version 2.0
6
- # (the "License"); you may not use this file except in compliance with
7
- # the License. You may obtain a copy of the License at
8
- #
9
- # http://www.apache.org/licenses/LICENSE-2.0
10
- #
11
- # Unless required by applicable law or agreed to in writing, software
12
- # distributed under the License is distributed on an "AS IS" BASIS,
13
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
- # See the License for the specific language governing permissions and
15
- # limitations under the License.
16
- #
17
-
18
- """
19
- PySpark is the Python API for Spark.
20
-
21
- Public classes:
22
-
23
- - :class:`SparkContext`:
24
- Main entry point for Spark functionality.
25
- - :class:`RDD`:
26
- A Resilient Distributed Dataset (RDD), the basic abstraction in Spark.
27
- - :class:`Broadcast`:
28
- A broadcast variable that gets reused across tasks.
29
- - :class:`Accumulator`:
30
- An "add-only" shared variable that tasks can only add values to.
31
- - :class:`SparkConf`:
32
- For configuring Spark.
33
- - :class:`SparkFiles`:
34
- Access files shipped with jobs.
35
- - :class:`StorageLevel`:
36
- Finer-grained cache persistence levels.
37
- - :class:`TaskContext`:
38
- Information about the current running task, available on the workers and experimental.
39
- - :class:`RDDBarrier`:
40
- Wraps an RDD under a barrier stage for barrier execution.
41
- - :class:`BarrierTaskContext`:
42
- A :class:`TaskContext` that provides extra info and tooling for barrier execution.
43
- - :class:`BarrierTaskInfo`:
44
- Information about a barrier task.
45
- - :class:`InheritableThread`:
46
- A inheritable thread to use in Spark when the pinned thread mode is on.
47
- """
48
-
49
- import sys
50
- from functools import wraps
51
- from typing import cast, Any, Callable, TypeVar, Union
52
-
53
- from pyspark.util import is_remote_only
54
-
55
- if not is_remote_only():
56
- from pyspark.core.rdd import RDD, RDDBarrier
57
- from pyspark.core.files import SparkFiles
58
- from pyspark.core.status import StatusTracker, SparkJobInfo, SparkStageInfo
59
- from pyspark.core.broadcast import Broadcast
60
- from pyspark.core import rdd, files, status, broadcast
61
-
62
- # for backward compatibility references.
63
- sys.modules["pyspark.rdd"] = rdd
64
- sys.modules["pyspark.files"] = files
65
- sys.modules["pyspark.status"] = status
66
- sys.modules["pyspark.broadcast"] = broadcast
67
-
68
- from pyspark.conf import SparkConf
69
- from pyspark.util import InheritableThread, inheritable_thread_target
70
- from pyspark.storagelevel import StorageLevel
71
- from pyspark.accumulators import Accumulator, AccumulatorParam
72
- from pyspark.serializers import MarshalSerializer, CPickleSerializer
73
- from pyspark.taskcontext import TaskContext, BarrierTaskContext, BarrierTaskInfo
74
- from pyspark.profiler import Profiler, BasicProfiler
75
- from pyspark.version import __version__
76
- from pyspark._globals import _NoValue # noqa: F401
77
-
78
- _F = TypeVar("_F", bound=Callable)
79
-
80
-
81
- def since(version: Union[str, float]) -> Callable[[_F], _F]:
82
- """
83
- A decorator that annotates a function to append the version of Spark the function was added.
84
- """
85
- import re
86
-
87
- indent_p = re.compile(r"\n( +)")
88
-
89
- def deco(f: _F) -> _F:
90
- assert f.__doc__ is not None
91
-
92
- indents = indent_p.findall(f.__doc__)
93
- indent = " " * (min(len(m) for m in indents) if indents else 0)
94
- f.__doc__ = f.__doc__.rstrip() + "\n\n%s.. versionadded:: %s" % (indent, version)
95
- return f
96
-
97
- return deco
98
-
99
-
100
- def keyword_only(func: _F) -> _F:
101
- """
102
- A decorator that forces keyword arguments in the wrapped method
103
- and saves actual input keyword arguments in `_input_kwargs`.
104
-
105
- Notes
106
- -----
107
- Should only be used to wrap a method where first arg is `self`
108
- """
109
-
110
- @wraps(func)
111
- def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any:
112
- if len(args) > 0:
113
- raise TypeError("Method %s forces keyword arguments." % func.__name__)
114
- self._input_kwargs = kwargs
115
- return func(self, **kwargs)
116
-
117
- return cast(_F, wrapper)
118
-
119
-
120
- # To avoid circular dependencies
121
- if not is_remote_only():
122
- from pyspark.core.context import SparkContext
123
- from pyspark.core import context
124
-
125
- # for backward compatibility references.
126
- sys.modules["pyspark.context"] = context
127
-
128
- # for back compatibility
129
- from pyspark.sql import SQLContext, HiveContext # noqa: F401
130
-
131
- from pyspark.sql import Row # noqa: F401
132
-
133
- __all__ = [
134
- "SparkConf",
135
- "SparkContext",
136
- "SparkFiles",
137
- "RDD",
138
- "StorageLevel",
139
- "Broadcast",
140
- "Accumulator",
141
- "AccumulatorParam",
142
- "MarshalSerializer",
143
- "CPickleSerializer",
144
- "StatusTracker",
145
- "SparkJobInfo",
146
- "SparkStageInfo",
147
- "Profiler",
148
- "BasicProfiler",
149
- "TaskContext",
150
- "RDDBarrier",
151
- "BarrierTaskContext",
152
- "BarrierTaskInfo",
153
- "InheritableThread",
154
- "inheritable_thread_target",
155
- "__version__",
156
- ]
@@ -1,71 +0,0 @@
1
- #
2
- # Licensed to the Apache Software Foundation (ASF) under one or more
3
- # contributor license agreements. See the NOTICE file distributed with
4
- # this work for additional information regarding copyright ownership.
5
- # The ASF licenses this file to You under the Apache License, Version 2.0
6
- # (the "License"); you may not use this file except in compliance with
7
- # the License. You may obtain a copy of the License at
8
- #
9
- # http://www.apache.org/licenses/LICENSE-2.0
10
- #
11
- # Unless required by applicable law or agreed to in writing, software
12
- # distributed under the License is distributed on an "AS IS" BASIS,
13
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
- # See the License for the specific language governing permissions and
15
- # limitations under the License.
16
- #
17
-
18
- """
19
- Module defining global singleton classes.
20
-
21
- This module raises a RuntimeError if an attempt to reload it is made. In that
22
- way the identities of the classes defined here are fixed and will remain so
23
- even if pyspark itself is reloaded. In particular, a function like the following
24
- will still work correctly after pyspark is reloaded:
25
-
26
- def foo(arg=pyspark._NoValue):
27
- if arg is pyspark._NoValue:
28
- ...
29
-
30
- See gh-7844 for a discussion of the reload problem that motivated this module.
31
-
32
- Note that this approach is taken after from NumPy.
33
- """
34
-
35
- __ALL__ = ["_NoValue"]
36
-
37
-
38
- # Disallow reloading this module so as to preserve the identities of the
39
- # classes defined here.
40
- if "_is_loaded" in globals():
41
- raise RuntimeError("Reloading pyspark._globals is not allowed")
42
- _is_loaded = True
43
-
44
-
45
- class _NoValueType:
46
- """Special keyword value.
47
-
48
- The instance of this class may be used as the default value assigned to a
49
- deprecated keyword in order to check if it has been given a user defined
50
- value.
51
-
52
- This class was copied from NumPy.
53
- """
54
-
55
- __instance = None
56
-
57
- def __new__(cls):
58
- # ensure that only one instance exists
59
- if not cls.__instance:
60
- cls.__instance = super(_NoValueType, cls).__new__(cls)
61
- return cls.__instance
62
-
63
- # needed for python 2 to preserve identity through a pickle
64
- def __reduce__(self):
65
- return (self.__class__, ())
66
-
67
- def __repr__(self):
68
- return "<no value>"
69
-
70
-
71
- _NoValue = _NoValueType()
@@ -1,43 +0,0 @@
1
- #
2
- # Licensed to the Apache Software Foundation (ASF) under one
3
- # or more contributor license agreements. See the NOTICE file
4
- # distributed with this work for additional information
5
- # regarding copyright ownership. The ASF licenses this file
6
- # to you under the Apache License, Version 2.0 (the
7
- # "License"); you may not use this file except in compliance
8
- # with the License. You may obtain a copy of the License at
9
- #
10
- # http://www.apache.org/licenses/LICENSE-2.0
11
- #
12
- # Unless required by applicable law or agreed to in writing,
13
- # software distributed under the License is distributed on an
14
- # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15
- # KIND, either express or implied. See the License for the
16
- # specific language governing permissions and limitations
17
- # under the License.
18
-
19
- from typing import Callable, Iterable, Sized, TypeVar, Union
20
- from typing_extensions import Literal, Protocol
21
-
22
- from numpy import int32, int64, float32, float64, ndarray
23
-
24
- F = TypeVar("F", bound=Callable)
25
- T_co = TypeVar("T_co", covariant=True)
26
-
27
- PrimitiveType = Union[bool, float, int, str]
28
-
29
- NonUDFType = Literal[0]
30
-
31
- class SupportsIAdd(Protocol):
32
- def __iadd__(self, other: SupportsIAdd) -> SupportsIAdd: ...
33
-
34
- class SupportsOrdering(Protocol):
35
- def __lt__(self, other: SupportsOrdering) -> bool: ...
36
-
37
- class SizedIterable(Protocol, Sized, Iterable[T_co]): ...
38
-
39
- S = TypeVar("S", bound=SupportsOrdering)
40
-
41
- NumberOrArray = TypeVar(
42
- "NumberOrArray", float, int, complex, int32, int64, float32, float64, ndarray
43
- )