pyspark-connect 4.0.0.dev2__tar.gz → 4.1.0.dev1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pyspark_connect-4.0.0.dev2/pyspark_connect.egg-info → pyspark_connect-4.1.0.dev1}/PKG-INFO +20 -7
- {pyspark_connect-4.0.0.dev2/pyspark/logger → pyspark_connect-4.1.0.dev1/pyspark_connect}/__init__.py +4 -5
- pyspark_connect-4.1.0.dev1/pyspark_connect/version.py +1 -0
- {pyspark_connect-4.0.0.dev2 → pyspark_connect-4.1.0.dev1/pyspark_connect.egg-info}/PKG-INFO +20 -7
- pyspark_connect-4.1.0.dev1/pyspark_connect.egg-info/SOURCES.txt +11 -0
- pyspark_connect-4.1.0.dev1/pyspark_connect.egg-info/requires.txt +8 -0
- pyspark_connect-4.1.0.dev1/pyspark_connect.egg-info/top_level.txt +1 -0
- {pyspark_connect-4.0.0.dev2 → pyspark_connect-4.1.0.dev1}/setup.py +34 -109
- pyspark_connect-4.0.0.dev2/pyspark/__init__.py +0 -156
- pyspark_connect-4.0.0.dev2/pyspark/_globals.py +0 -71
- pyspark_connect-4.0.0.dev2/pyspark/_typing.pyi +0 -43
- pyspark_connect-4.0.0.dev2/pyspark/accumulators.py +0 -345
- pyspark_connect-4.0.0.dev2/pyspark/cloudpickle/__init__.py +0 -18
- pyspark_connect-4.0.0.dev2/pyspark/cloudpickle/cloudpickle.py +0 -1487
- pyspark_connect-4.0.0.dev2/pyspark/cloudpickle/cloudpickle_fast.py +0 -13
- pyspark_connect-4.0.0.dev2/pyspark/cloudpickle/compat.py +0 -18
- pyspark_connect-4.0.0.dev2/pyspark/conf.py +0 -282
- pyspark_connect-4.0.0.dev2/pyspark/daemon.py +0 -225
- pyspark_connect-4.0.0.dev2/pyspark/errors/__init__.py +0 -88
- pyspark_connect-4.0.0.dev2/pyspark/errors/error-conditions.json +0 -1171
- pyspark_connect-4.0.0.dev2/pyspark/errors/error_classes.py +0 -31
- pyspark_connect-4.0.0.dev2/pyspark/errors/exceptions/__init__.py +0 -32
- pyspark_connect-4.0.0.dev2/pyspark/errors/exceptions/base.py +0 -418
- pyspark_connect-4.0.0.dev2/pyspark/errors/exceptions/captured.py +0 -445
- pyspark_connect-4.0.0.dev2/pyspark/errors/exceptions/connect.py +0 -532
- pyspark_connect-4.0.0.dev2/pyspark/errors/utils.py +0 -307
- pyspark_connect-4.0.0.dev2/pyspark/errors_doc_gen.py +0 -62
- pyspark_connect-4.0.0.dev2/pyspark/find_spark_home.py +0 -95
- pyspark_connect-4.0.0.dev2/pyspark/install.py +0 -203
- pyspark_connect-4.0.0.dev2/pyspark/instrumentation_utils.py +0 -187
- pyspark_connect-4.0.0.dev2/pyspark/java_gateway.py +0 -192
- pyspark_connect-4.0.0.dev2/pyspark/join.py +0 -119
- pyspark_connect-4.0.0.dev2/pyspark/logger/logger.py +0 -196
- pyspark_connect-4.0.0.dev2/pyspark/loose_version.py +0 -75
- pyspark_connect-4.0.0.dev2/pyspark/ml/__init__.py +0 -71
- pyspark_connect-4.0.0.dev2/pyspark/ml/_typing.pyi +0 -84
- pyspark_connect-4.0.0.dev2/pyspark/ml/base.py +0 -412
- pyspark_connect-4.0.0.dev2/pyspark/ml/classification.py +0 -4346
- pyspark_connect-4.0.0.dev2/pyspark/ml/clustering.py +0 -2188
- pyspark_connect-4.0.0.dev2/pyspark/ml/common.py +0 -163
- pyspark_connect-4.0.0.dev2/pyspark/ml/connect/__init__.py +0 -46
- pyspark_connect-4.0.0.dev2/pyspark/ml/connect/base.py +0 -345
- pyspark_connect-4.0.0.dev2/pyspark/ml/connect/classification.py +0 -392
- pyspark_connect-4.0.0.dev2/pyspark/ml/connect/evaluation.py +0 -291
- pyspark_connect-4.0.0.dev2/pyspark/ml/connect/feature.py +0 -411
- pyspark_connect-4.0.0.dev2/pyspark/ml/connect/functions.py +0 -33
- pyspark_connect-4.0.0.dev2/pyspark/ml/connect/io_utils.py +0 -334
- pyspark_connect-4.0.0.dev2/pyspark/ml/connect/pipeline.py +0 -263
- pyspark_connect-4.0.0.dev2/pyspark/ml/connect/summarizer.py +0 -121
- pyspark_connect-4.0.0.dev2/pyspark/ml/connect/tuning.py +0 -577
- pyspark_connect-4.0.0.dev2/pyspark/ml/connect/util.py +0 -175
- pyspark_connect-4.0.0.dev2/pyspark/ml/deepspeed/__init__.py +0 -16
- pyspark_connect-4.0.0.dev2/pyspark/ml/deepspeed/deepspeed_distributor.py +0 -164
- pyspark_connect-4.0.0.dev2/pyspark/ml/dl_util.py +0 -150
- pyspark_connect-4.0.0.dev2/pyspark/ml/evaluation.py +0 -1165
- pyspark_connect-4.0.0.dev2/pyspark/ml/feature.py +0 -7480
- pyspark_connect-4.0.0.dev2/pyspark/ml/fpm.py +0 -543
- pyspark_connect-4.0.0.dev2/pyspark/ml/functions.py +0 -868
- pyspark_connect-4.0.0.dev2/pyspark/ml/image.py +0 -267
- pyspark_connect-4.0.0.dev2/pyspark/ml/linalg/__init__.py +0 -1379
- pyspark_connect-4.0.0.dev2/pyspark/ml/model_cache.py +0 -55
- pyspark_connect-4.0.0.dev2/pyspark/ml/param/__init__.py +0 -604
- pyspark_connect-4.0.0.dev2/pyspark/ml/param/_shared_params_code_gen.py +0 -375
- pyspark_connect-4.0.0.dev2/pyspark/ml/param/shared.py +0 -900
- pyspark_connect-4.0.0.dev2/pyspark/ml/pipeline.py +0 -457
- pyspark_connect-4.0.0.dev2/pyspark/ml/recommendation.py +0 -748
- pyspark_connect-4.0.0.dev2/pyspark/ml/regression.py +0 -3333
- pyspark_connect-4.0.0.dev2/pyspark/ml/stat.py +0 -530
- pyspark_connect-4.0.0.dev2/pyspark/ml/torch/__init__.py +0 -16
- pyspark_connect-4.0.0.dev2/pyspark/ml/torch/data.py +0 -102
- pyspark_connect-4.0.0.dev2/pyspark/ml/torch/distributor.py +0 -1094
- pyspark_connect-4.0.0.dev2/pyspark/ml/torch/log_communication.py +0 -198
- pyspark_connect-4.0.0.dev2/pyspark/ml/torch/torch_run_process_wrapper.py +0 -83
- pyspark_connect-4.0.0.dev2/pyspark/ml/tree.py +0 -434
- pyspark_connect-4.0.0.dev2/pyspark/ml/tuning.py +0 -1750
- pyspark_connect-4.0.0.dev2/pyspark/ml/util.py +0 -758
- pyspark_connect-4.0.0.dev2/pyspark/ml/wrapper.py +0 -478
- pyspark_connect-4.0.0.dev2/pyspark/mllib/__init__.py +0 -44
- pyspark_connect-4.0.0.dev2/pyspark/mllib/_typing.pyi +0 -33
- pyspark_connect-4.0.0.dev2/pyspark/mllib/classification.py +0 -989
- pyspark_connect-4.0.0.dev2/pyspark/mllib/clustering.py +0 -1318
- pyspark_connect-4.0.0.dev2/pyspark/mllib/common.py +0 -174
- pyspark_connect-4.0.0.dev2/pyspark/mllib/evaluation.py +0 -690
- pyspark_connect-4.0.0.dev2/pyspark/mllib/feature.py +0 -1084
- pyspark_connect-4.0.0.dev2/pyspark/mllib/fpm.py +0 -233
- pyspark_connect-4.0.0.dev2/pyspark/mllib/linalg/__init__.py +0 -1650
- pyspark_connect-4.0.0.dev2/pyspark/mllib/linalg/distributed.py +0 -1662
- pyspark_connect-4.0.0.dev2/pyspark/mllib/random.py +0 -698
- pyspark_connect-4.0.0.dev2/pyspark/mllib/recommendation.py +0 -389
- pyspark_connect-4.0.0.dev2/pyspark/mllib/regression.py +0 -1067
- pyspark_connect-4.0.0.dev2/pyspark/mllib/stat/KernelDensity.py +0 -59
- pyspark_connect-4.0.0.dev2/pyspark/mllib/stat/__init__.py +0 -34
- pyspark_connect-4.0.0.dev2/pyspark/mllib/stat/_statistics.py +0 -409
- pyspark_connect-4.0.0.dev2/pyspark/mllib/stat/distribution.py +0 -39
- pyspark_connect-4.0.0.dev2/pyspark/mllib/stat/test.py +0 -86
- pyspark_connect-4.0.0.dev2/pyspark/mllib/tree.py +0 -888
- pyspark_connect-4.0.0.dev2/pyspark/mllib/util.py +0 -660
- pyspark_connect-4.0.0.dev2/pyspark/pandas/__init__.py +0 -154
- pyspark_connect-4.0.0.dev2/pyspark/pandas/_typing.py +0 -52
- pyspark_connect-4.0.0.dev2/pyspark/pandas/accessors.py +0 -988
- pyspark_connect-4.0.0.dev2/pyspark/pandas/base.py +0 -1715
- pyspark_connect-4.0.0.dev2/pyspark/pandas/categorical.py +0 -798
- pyspark_connect-4.0.0.dev2/pyspark/pandas/config.py +0 -548
- pyspark_connect-4.0.0.dev2/pyspark/pandas/correlation.py +0 -261
- pyspark_connect-4.0.0.dev2/pyspark/pandas/data_type_ops/__init__.py +0 -16
- pyspark_connect-4.0.0.dev2/pyspark/pandas/data_type_ops/base.py +0 -517
- pyspark_connect-4.0.0.dev2/pyspark/pandas/data_type_ops/binary_ops.py +0 -98
- pyspark_connect-4.0.0.dev2/pyspark/pandas/data_type_ops/boolean_ops.py +0 -420
- pyspark_connect-4.0.0.dev2/pyspark/pandas/data_type_ops/categorical_ops.py +0 -141
- pyspark_connect-4.0.0.dev2/pyspark/pandas/data_type_ops/complex_ops.py +0 -145
- pyspark_connect-4.0.0.dev2/pyspark/pandas/data_type_ops/date_ops.py +0 -121
- pyspark_connect-4.0.0.dev2/pyspark/pandas/data_type_ops/datetime_ops.py +0 -171
- pyspark_connect-4.0.0.dev2/pyspark/pandas/data_type_ops/null_ops.py +0 -83
- pyspark_connect-4.0.0.dev2/pyspark/pandas/data_type_ops/num_ops.py +0 -579
- pyspark_connect-4.0.0.dev2/pyspark/pandas/data_type_ops/string_ops.py +0 -154
- pyspark_connect-4.0.0.dev2/pyspark/pandas/data_type_ops/timedelta_ops.py +0 -101
- pyspark_connect-4.0.0.dev2/pyspark/pandas/data_type_ops/udt_ops.py +0 -29
- pyspark_connect-4.0.0.dev2/pyspark/pandas/datetimes.py +0 -925
- pyspark_connect-4.0.0.dev2/pyspark/pandas/exceptions.py +0 -140
- pyspark_connect-4.0.0.dev2/pyspark/pandas/extensions.py +0 -388
- pyspark_connect-4.0.0.dev2/pyspark/pandas/frame.py +0 -13814
- pyspark_connect-4.0.0.dev2/pyspark/pandas/generic.py +0 -3649
- pyspark_connect-4.0.0.dev2/pyspark/pandas/groupby.py +0 -4620
- pyspark_connect-4.0.0.dev2/pyspark/pandas/indexes/__init__.py +0 -20
- pyspark_connect-4.0.0.dev2/pyspark/pandas/indexes/base.py +0 -2676
- pyspark_connect-4.0.0.dev2/pyspark/pandas/indexes/category.py +0 -668
- pyspark_connect-4.0.0.dev2/pyspark/pandas/indexes/datetimes.py +0 -882
- pyspark_connect-4.0.0.dev2/pyspark/pandas/indexes/multi.py +0 -1292
- pyspark_connect-4.0.0.dev2/pyspark/pandas/indexes/timedelta.py +0 -205
- pyspark_connect-4.0.0.dev2/pyspark/pandas/indexing.py +0 -1864
- pyspark_connect-4.0.0.dev2/pyspark/pandas/internal.py +0 -1652
- pyspark_connect-4.0.0.dev2/pyspark/pandas/missing/__init__.py +0 -48
- pyspark_connect-4.0.0.dev2/pyspark/pandas/missing/common.py +0 -76
- pyspark_connect-4.0.0.dev2/pyspark/pandas/missing/frame.py +0 -62
- pyspark_connect-4.0.0.dev2/pyspark/pandas/missing/general_functions.py +0 -43
- pyspark_connect-4.0.0.dev2/pyspark/pandas/missing/groupby.py +0 -92
- pyspark_connect-4.0.0.dev2/pyspark/pandas/missing/indexes.py +0 -184
- pyspark_connect-4.0.0.dev2/pyspark/pandas/missing/resample.py +0 -101
- pyspark_connect-4.0.0.dev2/pyspark/pandas/missing/scalars.py +0 -29
- pyspark_connect-4.0.0.dev2/pyspark/pandas/missing/series.py +0 -68
- pyspark_connect-4.0.0.dev2/pyspark/pandas/missing/window.py +0 -168
- pyspark_connect-4.0.0.dev2/pyspark/pandas/mlflow.py +0 -236
- pyspark_connect-4.0.0.dev2/pyspark/pandas/namespace.py +0 -3852
- pyspark_connect-4.0.0.dev2/pyspark/pandas/numpy_compat.py +0 -259
- pyspark_connect-4.0.0.dev2/pyspark/pandas/plot/__init__.py +0 -17
- pyspark_connect-4.0.0.dev2/pyspark/pandas/plot/core.py +0 -1185
- pyspark_connect-4.0.0.dev2/pyspark/pandas/plot/matplotlib.py +0 -972
- pyspark_connect-4.0.0.dev2/pyspark/pandas/plot/plotly.py +0 -255
- pyspark_connect-4.0.0.dev2/pyspark/pandas/resample.py +0 -791
- pyspark_connect-4.0.0.dev2/pyspark/pandas/series.py +0 -7358
- pyspark_connect-4.0.0.dev2/pyspark/pandas/spark/__init__.py +0 -16
- pyspark_connect-4.0.0.dev2/pyspark/pandas/spark/accessors.py +0 -1274
- pyspark_connect-4.0.0.dev2/pyspark/pandas/spark/functions.py +0 -225
- pyspark_connect-4.0.0.dev2/pyspark/pandas/spark/utils.py +0 -202
- pyspark_connect-4.0.0.dev2/pyspark/pandas/sql_formatter.py +0 -335
- pyspark_connect-4.0.0.dev2/pyspark/pandas/sql_processor.py +0 -390
- pyspark_connect-4.0.0.dev2/pyspark/pandas/strings.py +0 -2365
- pyspark_connect-4.0.0.dev2/pyspark/pandas/supported_api_gen.py +0 -439
- pyspark_connect-4.0.0.dev2/pyspark/pandas/testing.py +0 -328
- pyspark_connect-4.0.0.dev2/pyspark/pandas/typedef/__init__.py +0 -18
- pyspark_connect-4.0.0.dev2/pyspark/pandas/typedef/typehints.py +0 -877
- pyspark_connect-4.0.0.dev2/pyspark/pandas/usage_logging/__init__.py +0 -140
- pyspark_connect-4.0.0.dev2/pyspark/pandas/usage_logging/usage_logger.py +0 -132
- pyspark_connect-4.0.0.dev2/pyspark/pandas/utils.py +0 -1081
- pyspark_connect-4.0.0.dev2/pyspark/pandas/window.py +0 -2704
- pyspark_connect-4.0.0.dev2/pyspark/profiler.py +0 -526
- pyspark_connect-4.0.0.dev2/pyspark/py.typed +0 -1
- pyspark_connect-4.0.0.dev2/pyspark/rddsampler.py +0 -115
- pyspark_connect-4.0.0.dev2/pyspark/resource/__init__.py +0 -38
- pyspark_connect-4.0.0.dev2/pyspark/resource/information.py +0 -69
- pyspark_connect-4.0.0.dev2/pyspark/resource/profile.py +0 -342
- pyspark_connect-4.0.0.dev2/pyspark/resource/requests.py +0 -550
- pyspark_connect-4.0.0.dev2/pyspark/resultiterable.py +0 -45
- pyspark_connect-4.0.0.dev2/pyspark/serializers.py +0 -682
- pyspark_connect-4.0.0.dev2/pyspark/shell.py +0 -142
- pyspark_connect-4.0.0.dev2/pyspark/shuffle.py +0 -854
- pyspark_connect-4.0.0.dev2/pyspark/sql/__init__.py +0 -80
- pyspark_connect-4.0.0.dev2/pyspark/sql/_typing.pyi +0 -85
- pyspark_connect-4.0.0.dev2/pyspark/sql/avro/__init__.py +0 -18
- pyspark_connect-4.0.0.dev2/pyspark/sql/avro/functions.py +0 -224
- pyspark_connect-4.0.0.dev2/pyspark/sql/catalog.py +0 -1268
- pyspark_connect-4.0.0.dev2/pyspark/sql/column.py +0 -1557
- pyspark_connect-4.0.0.dev2/pyspark/sql/conf.py +0 -175
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/__init__.py +0 -18
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/_typing.py +0 -92
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/avro/__init__.py +0 -18
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/avro/functions.py +0 -139
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/catalog.py +0 -365
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/client/__init__.py +0 -23
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/client/artifact.py +0 -444
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/client/core.py +0 -1920
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/client/logging.py +0 -61
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/client/reattach.py +0 -334
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/client/retries.py +0 -317
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/column.py +0 -600
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/conf.py +0 -153
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/conversion.py +0 -584
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/dataframe.py +0 -2344
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/datasource.py +0 -47
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/expressions.py +0 -1142
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/functions/__init__.py +0 -21
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/functions/builtin.py +0 -4203
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/functions/partitioning.py +0 -112
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/group.py +0 -496
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/merge.py +0 -253
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/observation.py +0 -113
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/plan.py +0 -2503
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/profiler.py +0 -41
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/__init__.py +0 -25
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/base_pb2.py +0 -230
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/base_pb2.pyi +0 -3736
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/base_pb2_grpc.py +0 -517
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/catalog_pb2.py +0 -110
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/catalog_pb2.pyi +0 -1133
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/commands_pb2.py +0 -174
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/commands_pb2.pyi +0 -2306
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/common_pb2.py +0 -66
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/common_pb2.pyi +0 -349
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/example_plugins_pb2.py +0 -54
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/example_plugins_pb2.pyi +0 -112
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/expressions_pb2.py +0 -123
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/expressions_pb2.pyi +0 -1803
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/relations_pb2.py +0 -223
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/relations_pb2.pyi +0 -4072
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/types_pb2.py +0 -98
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/proto/types_pb2.pyi +0 -1012
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/protobuf/__init__.py +0 -18
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/protobuf/functions.py +0 -165
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/readwriter.py +0 -1015
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/resource/__init__.py +0 -16
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/resource/profile.py +0 -69
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/session.py +0 -1137
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/shell/__init__.py +0 -26
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/shell/progress.py +0 -212
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/sql_formatter.py +0 -76
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/streaming/__init__.py +0 -16
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/streaming/query.py +0 -464
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/streaming/readwriter.py +0 -720
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/streaming/worker/__init__.py +0 -18
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/streaming/worker/foreach_batch_worker.py +0 -101
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/streaming/worker/listener_worker.py +0 -115
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/types.py +0 -380
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/udf.py +0 -319
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/udtf.py +0 -217
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/utils.py +0 -100
- pyspark_connect-4.0.0.dev2/pyspark/sql/connect/window.py +0 -189
- pyspark_connect-4.0.0.dev2/pyspark/sql/context.py +0 -816
- pyspark_connect-4.0.0.dev2/pyspark/sql/dataframe.py +0 -6550
- pyspark_connect-4.0.0.dev2/pyspark/sql/datasource.py +0 -822
- pyspark_connect-4.0.0.dev2/pyspark/sql/datasource_internal.py +0 -146
- pyspark_connect-4.0.0.dev2/pyspark/sql/functions/__init__.py +0 -21
- pyspark_connect-4.0.0.dev2/pyspark/sql/functions/builtin.py +0 -21929
- pyspark_connect-4.0.0.dev2/pyspark/sql/functions/partitioning.py +0 -252
- pyspark_connect-4.0.0.dev2/pyspark/sql/group.py +0 -551
- pyspark_connect-4.0.0.dev2/pyspark/sql/merge.py +0 -253
- pyspark_connect-4.0.0.dev2/pyspark/sql/metrics.py +0 -287
- pyspark_connect-4.0.0.dev2/pyspark/sql/observation.py +0 -170
- pyspark_connect-4.0.0.dev2/pyspark/sql/pandas/__init__.py +0 -21
- pyspark_connect-4.0.0.dev2/pyspark/sql/pandas/_typing/__init__.pyi +0 -356
- pyspark_connect-4.0.0.dev2/pyspark/sql/pandas/_typing/protocols/__init__.pyi +0 -17
- pyspark_connect-4.0.0.dev2/pyspark/sql/pandas/_typing/protocols/frame.pyi +0 -20
- pyspark_connect-4.0.0.dev2/pyspark/sql/pandas/_typing/protocols/series.pyi +0 -20
- pyspark_connect-4.0.0.dev2/pyspark/sql/pandas/conversion.py +0 -838
- pyspark_connect-4.0.0.dev2/pyspark/sql/pandas/functions.py +0 -545
- pyspark_connect-4.0.0.dev2/pyspark/sql/pandas/functions.pyi +0 -132
- pyspark_connect-4.0.0.dev2/pyspark/sql/pandas/group_ops.py +0 -910
- pyspark_connect-4.0.0.dev2/pyspark/sql/pandas/map_ops.py +0 -127
- pyspark_connect-4.0.0.dev2/pyspark/sql/pandas/serializers.py +0 -1192
- pyspark_connect-4.0.0.dev2/pyspark/sql/pandas/typehints.py +0 -178
- pyspark_connect-4.0.0.dev2/pyspark/sql/pandas/types.py +0 -1346
- pyspark_connect-4.0.0.dev2/pyspark/sql/pandas/utils.py +0 -96
- pyspark_connect-4.0.0.dev2/pyspark/sql/profiler.py +0 -505
- pyspark_connect-4.0.0.dev2/pyspark/sql/protobuf/__init__.py +0 -18
- pyspark_connect-4.0.0.dev2/pyspark/sql/protobuf/functions.py +0 -336
- pyspark_connect-4.0.0.dev2/pyspark/sql/readwriter.py +0 -2544
- pyspark_connect-4.0.0.dev2/pyspark/sql/session.py +0 -2340
- pyspark_connect-4.0.0.dev2/pyspark/sql/sql_formatter.py +0 -86
- pyspark_connect-4.0.0.dev2/pyspark/sql/streaming/StateMessage_pb2.py +0 -73
- pyspark_connect-4.0.0.dev2/pyspark/sql/streaming/StateMessage_pb2.pyi +0 -179
- pyspark_connect-4.0.0.dev2/pyspark/sql/streaming/__init__.py +0 -25
- pyspark_connect-4.0.0.dev2/pyspark/sql/streaming/listener.py +0 -1112
- pyspark_connect-4.0.0.dev2/pyspark/sql/streaming/python_streaming_source_runner.py +0 -217
- pyspark_connect-4.0.0.dev2/pyspark/sql/streaming/query.py +0 -751
- pyspark_connect-4.0.0.dev2/pyspark/sql/streaming/readwriter.py +0 -1826
- pyspark_connect-4.0.0.dev2/pyspark/sql/streaming/state.py +0 -297
- pyspark_connect-4.0.0.dev2/pyspark/sql/streaming/stateful_processor.py +0 -168
- pyspark_connect-4.0.0.dev2/pyspark/sql/streaming/stateful_processor_api_client.py +0 -170
- pyspark_connect-4.0.0.dev2/pyspark/sql/streaming/value_state_client.py +0 -105
- pyspark_connect-4.0.0.dev2/pyspark/sql/types.py +0 -3332
- pyspark_connect-4.0.0.dev2/pyspark/sql/udf.py +0 -786
- pyspark_connect-4.0.0.dev2/pyspark/sql/udtf.py +0 -505
- pyspark_connect-4.0.0.dev2/pyspark/sql/utils.py +0 -434
- pyspark_connect-4.0.0.dev2/pyspark/sql/variant_utils.py +0 -726
- pyspark_connect-4.0.0.dev2/pyspark/sql/window.py +0 -429
- pyspark_connect-4.0.0.dev2/pyspark/sql/worker/__init__.py +0 -16
- pyspark_connect-4.0.0.dev2/pyspark/sql/worker/analyze_udtf.py +0 -270
- pyspark_connect-4.0.0.dev2/pyspark/sql/worker/commit_data_source_write.py +0 -115
- pyspark_connect-4.0.0.dev2/pyspark/sql/worker/create_data_source.py +0 -192
- pyspark_connect-4.0.0.dev2/pyspark/sql/worker/lookup_data_sources.py +0 -100
- pyspark_connect-4.0.0.dev2/pyspark/sql/worker/plan_data_source_read.py +0 -373
- pyspark_connect-4.0.0.dev2/pyspark/sql/worker/python_streaming_sink_runner.py +0 -151
- pyspark_connect-4.0.0.dev2/pyspark/sql/worker/write_into_data_source.py +0 -247
- pyspark_connect-4.0.0.dev2/pyspark/statcounter.py +0 -165
- pyspark_connect-4.0.0.dev2/pyspark/storagelevel.py +0 -97
- pyspark_connect-4.0.0.dev2/pyspark/streaming/__init__.py +0 -22
- pyspark_connect-4.0.0.dev2/pyspark/streaming/context.py +0 -471
- pyspark_connect-4.0.0.dev2/pyspark/streaming/dstream.py +0 -931
- pyspark_connect-4.0.0.dev2/pyspark/streaming/kinesis.py +0 -205
- pyspark_connect-4.0.0.dev2/pyspark/streaming/listener.py +0 -83
- pyspark_connect-4.0.0.dev2/pyspark/streaming/util.py +0 -168
- pyspark_connect-4.0.0.dev2/pyspark/taskcontext.py +0 -506
- pyspark_connect-4.0.0.dev2/pyspark/testing/__init__.py +0 -19
- pyspark_connect-4.0.0.dev2/pyspark/testing/connectutils.py +0 -222
- pyspark_connect-4.0.0.dev2/pyspark/testing/mllibutils.py +0 -30
- pyspark_connect-4.0.0.dev2/pyspark/testing/mlutils.py +0 -279
- pyspark_connect-4.0.0.dev2/pyspark/testing/pandasutils.py +0 -645
- pyspark_connect-4.0.0.dev2/pyspark/testing/sqlutils.py +0 -295
- pyspark_connect-4.0.0.dev2/pyspark/testing/streamingutils.py +0 -177
- pyspark_connect-4.0.0.dev2/pyspark/testing/utils.py +0 -1016
- pyspark_connect-4.0.0.dev2/pyspark/traceback_utils.py +0 -81
- pyspark_connect-4.0.0.dev2/pyspark/util.py +0 -801
- pyspark_connect-4.0.0.dev2/pyspark/version.py +0 -1
- pyspark_connect-4.0.0.dev2/pyspark/worker.py +0 -1938
- pyspark_connect-4.0.0.dev2/pyspark/worker_util.py +0 -189
- pyspark_connect-4.0.0.dev2/pyspark_connect.egg-info/SOURCES.txt +0 -326
- pyspark_connect-4.0.0.dev2/pyspark_connect.egg-info/requires.txt +0 -6
- pyspark_connect-4.0.0.dev2/pyspark_connect.egg-info/top_level.txt +0 -1
- {pyspark_connect-4.0.0.dev2 → pyspark_connect-4.1.0.dev1}/MANIFEST.in +0 -0
- {pyspark_connect-4.0.0.dev2 → pyspark_connect-4.1.0.dev1}/README.md +0 -0
- {pyspark_connect-4.0.0.dev2 → pyspark_connect-4.1.0.dev1}/pyspark_connect.egg-info/dependency_links.txt +0 -0
- {pyspark_connect-4.0.0.dev2 → pyspark_connect-4.1.0.dev1}/setup.cfg +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: pyspark-connect
|
|
3
|
-
Version: 4.
|
|
4
|
-
Summary: Python Spark Connect
|
|
3
|
+
Version: 4.1.0.dev1
|
|
4
|
+
Summary: Apache Spark Python API with Spark Connect by default
|
|
5
5
|
Home-page: https://github.com/apache/spark/tree/master/python
|
|
6
6
|
Author: Spark Developers
|
|
7
7
|
Author-email: dev@spark.apache.org
|
|
@@ -12,17 +12,30 @@ Classifier: Programming Language :: Python :: 3.9
|
|
|
12
12
|
Classifier: Programming Language :: Python :: 3.10
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.11
|
|
14
14
|
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
16
|
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
16
17
|
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
17
18
|
Classifier: Typing :: Typed
|
|
18
19
|
Requires-Python: >=3.9
|
|
19
20
|
Description-Content-Type: text/markdown
|
|
21
|
+
Requires-Dist: pyspark==4.1.0.dev1
|
|
20
22
|
Requires-Dist: pandas>=2.0.0
|
|
21
|
-
Requires-Dist: pyarrow>=
|
|
22
|
-
Requires-Dist: grpcio>=1.
|
|
23
|
-
Requires-Dist: grpcio-status>=1.
|
|
24
|
-
Requires-Dist: googleapis-common-protos>=1.
|
|
23
|
+
Requires-Dist: pyarrow>=11.0.0
|
|
24
|
+
Requires-Dist: grpcio>=1.67.0
|
|
25
|
+
Requires-Dist: grpcio-status>=1.67.0
|
|
26
|
+
Requires-Dist: googleapis-common-protos>=1.65.0
|
|
25
27
|
Requires-Dist: numpy>=1.21
|
|
28
|
+
Requires-Dist: pyyaml>=3.11
|
|
29
|
+
Dynamic: author
|
|
30
|
+
Dynamic: author-email
|
|
31
|
+
Dynamic: classifier
|
|
32
|
+
Dynamic: description
|
|
33
|
+
Dynamic: description-content-type
|
|
34
|
+
Dynamic: home-page
|
|
35
|
+
Dynamic: license
|
|
36
|
+
Dynamic: requires-dist
|
|
37
|
+
Dynamic: requires-python
|
|
38
|
+
Dynamic: summary
|
|
26
39
|
|
|
27
40
|
# Apache Spark
|
|
28
41
|
|
{pyspark_connect-4.0.0.dev2/pyspark/logger → pyspark_connect-4.1.0.dev1/pyspark_connect}/__init__.py
RENAMED
|
@@ -16,12 +16,11 @@
|
|
|
16
16
|
#
|
|
17
17
|
|
|
18
18
|
"""
|
|
19
|
-
|
|
19
|
+
A marker package to make Spark Connect default.
|
|
20
20
|
"""
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
)
|
|
21
|
+
|
|
22
|
+
from pyspark_connect.version import __version__
|
|
24
23
|
|
|
25
24
|
__all__ = [
|
|
26
|
-
"
|
|
25
|
+
"__version__",
|
|
27
26
|
]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__: str = '4.1.0.dev1'
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: pyspark-connect
|
|
3
|
-
Version: 4.
|
|
4
|
-
Summary: Python Spark Connect
|
|
3
|
+
Version: 4.1.0.dev1
|
|
4
|
+
Summary: Apache Spark Python API with Spark Connect by default
|
|
5
5
|
Home-page: https://github.com/apache/spark/tree/master/python
|
|
6
6
|
Author: Spark Developers
|
|
7
7
|
Author-email: dev@spark.apache.org
|
|
@@ -12,17 +12,30 @@ Classifier: Programming Language :: Python :: 3.9
|
|
|
12
12
|
Classifier: Programming Language :: Python :: 3.10
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.11
|
|
14
14
|
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
16
|
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
16
17
|
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
17
18
|
Classifier: Typing :: Typed
|
|
18
19
|
Requires-Python: >=3.9
|
|
19
20
|
Description-Content-Type: text/markdown
|
|
21
|
+
Requires-Dist: pyspark==4.1.0.dev1
|
|
20
22
|
Requires-Dist: pandas>=2.0.0
|
|
21
|
-
Requires-Dist: pyarrow>=
|
|
22
|
-
Requires-Dist: grpcio>=1.
|
|
23
|
-
Requires-Dist: grpcio-status>=1.
|
|
24
|
-
Requires-Dist: googleapis-common-protos>=1.
|
|
23
|
+
Requires-Dist: pyarrow>=11.0.0
|
|
24
|
+
Requires-Dist: grpcio>=1.67.0
|
|
25
|
+
Requires-Dist: grpcio-status>=1.67.0
|
|
26
|
+
Requires-Dist: googleapis-common-protos>=1.65.0
|
|
25
27
|
Requires-Dist: numpy>=1.21
|
|
28
|
+
Requires-Dist: pyyaml>=3.11
|
|
29
|
+
Dynamic: author
|
|
30
|
+
Dynamic: author-email
|
|
31
|
+
Dynamic: classifier
|
|
32
|
+
Dynamic: description
|
|
33
|
+
Dynamic: description-content-type
|
|
34
|
+
Dynamic: home-page
|
|
35
|
+
Dynamic: license
|
|
36
|
+
Dynamic: requires-dist
|
|
37
|
+
Dynamic: requires-python
|
|
38
|
+
Dynamic: summary
|
|
26
39
|
|
|
27
40
|
# Apache Spark
|
|
28
41
|
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
MANIFEST.in
|
|
2
|
+
README.md
|
|
3
|
+
setup.cfg
|
|
4
|
+
setup.py
|
|
5
|
+
pyspark_connect/__init__.py
|
|
6
|
+
pyspark_connect/version.py
|
|
7
|
+
pyspark_connect.egg-info/PKG-INFO
|
|
8
|
+
pyspark_connect.egg-info/SOURCES.txt
|
|
9
|
+
pyspark_connect.egg-info/dependency_links.txt
|
|
10
|
+
pyspark_connect.egg-info/requires.txt
|
|
11
|
+
pyspark_connect.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
pyspark_connect
|
|
@@ -17,20 +17,20 @@
|
|
|
17
17
|
# limitations under the License.
|
|
18
18
|
|
|
19
19
|
# cd python
|
|
20
|
-
# python packaging/
|
|
20
|
+
# python packaging/connect/setup.py sdist
|
|
21
21
|
|
|
22
|
-
# cd python/packaging/
|
|
22
|
+
# cd python/packaging/connect
|
|
23
23
|
# python setup.py sdist
|
|
24
24
|
|
|
25
25
|
import sys
|
|
26
26
|
from setuptools import setup
|
|
27
27
|
import os
|
|
28
|
-
from shutil import copyfile, move
|
|
28
|
+
from shutil import copyfile, copytree, move, rmtree
|
|
29
29
|
import glob
|
|
30
30
|
from pathlib import Path
|
|
31
31
|
|
|
32
32
|
if (
|
|
33
|
-
# When we package, the parent
|
|
33
|
+
# When we package, the parent directory 'connect' dir
|
|
34
34
|
# (as we pip install -e python/packaging/connect)
|
|
35
35
|
os.getcwd() == str(Path(__file__).parent.absolute())
|
|
36
36
|
and str(Path(__file__).parent.name) == "connect"
|
|
@@ -47,16 +47,6 @@ if (
|
|
|
47
47
|
# Move to spark/python
|
|
48
48
|
os.chdir(Path(__file__).parent.parent.parent.absolute())
|
|
49
49
|
|
|
50
|
-
try:
|
|
51
|
-
exec(open("pyspark/version.py").read())
|
|
52
|
-
except IOError:
|
|
53
|
-
print(
|
|
54
|
-
"Failed to load PySpark version file for packaging. You must be in Spark's python dir.",
|
|
55
|
-
file=sys.stderr,
|
|
56
|
-
)
|
|
57
|
-
sys.exit(-1)
|
|
58
|
-
VERSION = __version__ # noqa
|
|
59
|
-
|
|
60
50
|
# Check and see if we are under the spark path in which case we need to build the symlink farm.
|
|
61
51
|
# This is important because we only want to build the symlink farm while under Spark otherwise we
|
|
62
52
|
# want to use the symlink farm. And if the symlink farm exists under while under Spark (e.g. a
|
|
@@ -65,142 +55,74 @@ in_spark = os.path.isfile("../core/src/main/scala/org/apache/spark/SparkContext.
|
|
|
65
55
|
os.path.isfile("../RELEASE") and len(glob.glob("../jars/spark*core*.jar")) == 1
|
|
66
56
|
)
|
|
67
57
|
|
|
68
|
-
test_packages = []
|
|
69
|
-
if "SPARK_TESTING" in os.environ:
|
|
70
|
-
test_packages = [
|
|
71
|
-
"pyspark.tests", # for Memory profiler parity tests
|
|
72
|
-
"pyspark.testing",
|
|
73
|
-
"pyspark.resource.tests",
|
|
74
|
-
"pyspark.sql.tests",
|
|
75
|
-
"pyspark.sql.tests.connect",
|
|
76
|
-
"pyspark.sql.tests.connect.streaming",
|
|
77
|
-
"pyspark.sql.tests.connect.client",
|
|
78
|
-
"pyspark.sql.tests.connect.shell",
|
|
79
|
-
"pyspark.sql.tests.pandas",
|
|
80
|
-
"pyspark.sql.tests.streaming",
|
|
81
|
-
"pyspark.ml.tests.connect",
|
|
82
|
-
"pyspark.pandas.tests",
|
|
83
|
-
"pyspark.pandas.tests.computation",
|
|
84
|
-
"pyspark.pandas.tests.data_type_ops",
|
|
85
|
-
"pyspark.pandas.tests.diff_frames_ops",
|
|
86
|
-
"pyspark.pandas.tests.frame",
|
|
87
|
-
"pyspark.pandas.tests.groupby",
|
|
88
|
-
"pyspark.pandas.tests.indexes",
|
|
89
|
-
"pyspark.pandas.tests.io",
|
|
90
|
-
"pyspark.pandas.tests.plot",
|
|
91
|
-
"pyspark.pandas.tests.resample",
|
|
92
|
-
"pyspark.pandas.tests.reshape",
|
|
93
|
-
"pyspark.pandas.tests.series",
|
|
94
|
-
"pyspark.pandas.tests.window",
|
|
95
|
-
"pyspark.pandas.tests.connect",
|
|
96
|
-
"pyspark.pandas.tests.connect.computation",
|
|
97
|
-
"pyspark.pandas.tests.connect.data_type_ops",
|
|
98
|
-
"pyspark.pandas.tests.connect.diff_frames_ops",
|
|
99
|
-
"pyspark.pandas.tests.connect.frame",
|
|
100
|
-
"pyspark.pandas.tests.connect.groupby",
|
|
101
|
-
"pyspark.pandas.tests.connect.indexes",
|
|
102
|
-
"pyspark.pandas.tests.connect.io",
|
|
103
|
-
"pyspark.pandas.tests.connect.plot",
|
|
104
|
-
"pyspark.pandas.tests.connect.resample",
|
|
105
|
-
"pyspark.pandas.tests.connect.reshape",
|
|
106
|
-
"pyspark.pandas.tests.connect.series",
|
|
107
|
-
"pyspark.pandas.tests.connect.window",
|
|
108
|
-
"pyspark.logger.tests",
|
|
109
|
-
"pyspark.logger.tests.connect",
|
|
110
|
-
]
|
|
111
|
-
|
|
112
58
|
try:
|
|
113
59
|
if in_spark:
|
|
114
60
|
# !!HACK ALTERT!!
|
|
115
61
|
# 1. `setup.py` has to be located with the same directory with the package.
|
|
116
62
|
# Therefore, we copy the current file, and place it at `spark/python` directory.
|
|
117
63
|
# After that, we remove it in the end.
|
|
118
|
-
# 2. Here it renames `lib` to `lib.
|
|
119
|
-
# We rename it back in the end.
|
|
64
|
+
# 2. Here it renames `pyspark` and `lib` to `pyspark.back` and `lib.back` so MANIFEST.in
|
|
65
|
+
# does not pick `pyspark` and `py4j` up. We rename it back in the end.
|
|
66
|
+
move("pyspark", "pyspark.back")
|
|
120
67
|
move("lib", "lib.back")
|
|
121
68
|
copyfile("packaging/connect/setup.py", "setup.py")
|
|
122
69
|
copyfile("packaging/connect/setup.cfg", "setup.cfg")
|
|
70
|
+
copytree("packaging/connect/pyspark_connect", "pyspark_connect")
|
|
71
|
+
copyfile("pyspark.back/version.py", "pyspark_connect/version.py")
|
|
72
|
+
|
|
73
|
+
try:
|
|
74
|
+
exec(open("pyspark_connect/version.py").read())
|
|
75
|
+
except IOError:
|
|
76
|
+
print(
|
|
77
|
+
"Failed to load PySpark Connect version file for packaging. "
|
|
78
|
+
"You must be in Spark's python dir.",
|
|
79
|
+
file=sys.stderr,
|
|
80
|
+
)
|
|
81
|
+
sys.exit(-1)
|
|
82
|
+
VERSION = __version__ # noqa
|
|
123
83
|
|
|
124
84
|
# If you are changing the versions here, please also change ./python/pyspark/sql/pandas/utils.py
|
|
125
85
|
# For Arrow, you should also check ./pom.xml and ensure there are no breaking changes in the
|
|
126
86
|
# binary format protocol with the Java version, see ARROW_HOME/format/* for specifications.
|
|
127
|
-
# Also don't forget to update python/docs/source/getting_started/install.rst,
|
|
128
|
-
# python/packaging/classic/setup.py
|
|
87
|
+
# Also don't forget to update python/docs/source/getting_started/install.rst,
|
|
88
|
+
# python/packaging/classic/setup.py, and python/packaging/client/setup.py
|
|
129
89
|
_minimum_pandas_version = "2.0.0"
|
|
130
90
|
_minimum_numpy_version = "1.21"
|
|
131
|
-
_minimum_pyarrow_version = "
|
|
132
|
-
_minimum_grpc_version = "1.
|
|
133
|
-
_minimum_googleapis_common_protos_version = "1.
|
|
91
|
+
_minimum_pyarrow_version = "11.0.0"
|
|
92
|
+
_minimum_grpc_version = "1.67.0"
|
|
93
|
+
_minimum_googleapis_common_protos_version = "1.65.0"
|
|
94
|
+
_minimum_pyyaml_version = "3.11"
|
|
134
95
|
|
|
135
96
|
with open("README.md") as f:
|
|
136
97
|
long_description = f.read()
|
|
137
98
|
|
|
138
99
|
connect_packages = [
|
|
139
|
-
"
|
|
140
|
-
"pyspark.cloudpickle",
|
|
141
|
-
"pyspark.mllib",
|
|
142
|
-
"pyspark.mllib.linalg",
|
|
143
|
-
"pyspark.mllib.stat",
|
|
144
|
-
"pyspark.ml",
|
|
145
|
-
"pyspark.ml.connect",
|
|
146
|
-
"pyspark.ml.linalg",
|
|
147
|
-
"pyspark.ml.param",
|
|
148
|
-
"pyspark.ml.torch",
|
|
149
|
-
"pyspark.ml.deepspeed",
|
|
150
|
-
"pyspark.sql",
|
|
151
|
-
"pyspark.sql.avro",
|
|
152
|
-
"pyspark.sql.connect",
|
|
153
|
-
"pyspark.sql.connect.avro",
|
|
154
|
-
"pyspark.sql.connect.client",
|
|
155
|
-
"pyspark.sql.connect.functions",
|
|
156
|
-
"pyspark.sql.connect.proto",
|
|
157
|
-
"pyspark.sql.connect.protobuf",
|
|
158
|
-
"pyspark.sql.connect.resource",
|
|
159
|
-
"pyspark.sql.connect.shell",
|
|
160
|
-
"pyspark.sql.connect.streaming",
|
|
161
|
-
"pyspark.sql.connect.streaming.worker",
|
|
162
|
-
"pyspark.sql.functions",
|
|
163
|
-
"pyspark.sql.pandas",
|
|
164
|
-
"pyspark.sql.protobuf",
|
|
165
|
-
"pyspark.sql.streaming",
|
|
166
|
-
"pyspark.sql.worker",
|
|
167
|
-
"pyspark.streaming",
|
|
168
|
-
"pyspark.pandas",
|
|
169
|
-
"pyspark.pandas.data_type_ops",
|
|
170
|
-
"pyspark.pandas.indexes",
|
|
171
|
-
"pyspark.pandas.missing",
|
|
172
|
-
"pyspark.pandas.plot",
|
|
173
|
-
"pyspark.pandas.spark",
|
|
174
|
-
"pyspark.pandas.typedef",
|
|
175
|
-
"pyspark.pandas.usage_logging",
|
|
176
|
-
"pyspark.testing",
|
|
177
|
-
"pyspark.resource",
|
|
178
|
-
"pyspark.errors",
|
|
179
|
-
"pyspark.errors.exceptions",
|
|
180
|
-
"pyspark.logger",
|
|
100
|
+
"pyspark_connect",
|
|
181
101
|
]
|
|
182
102
|
|
|
183
103
|
setup(
|
|
184
104
|
name="pyspark-connect",
|
|
185
105
|
version=VERSION,
|
|
186
|
-
description="Python Spark Connect
|
|
106
|
+
description="Apache Spark Python API with Spark Connect by default",
|
|
187
107
|
long_description=long_description,
|
|
188
108
|
long_description_content_type="text/markdown",
|
|
189
109
|
author="Spark Developers",
|
|
190
110
|
author_email="dev@spark.apache.org",
|
|
191
111
|
url="https://github.com/apache/spark/tree/master/python",
|
|
192
|
-
packages=connect_packages
|
|
112
|
+
packages=connect_packages,
|
|
193
113
|
include_package_data=True,
|
|
194
114
|
license="http://www.apache.org/licenses/LICENSE-2.0",
|
|
195
115
|
# Don't forget to update python/docs/source/getting_started/install.rst
|
|
196
116
|
# if you're updating the versions or dependencies.
|
|
197
117
|
install_requires=[
|
|
118
|
+
"pyspark==%s" % VERSION,
|
|
198
119
|
"pandas>=%s" % _minimum_pandas_version,
|
|
199
120
|
"pyarrow>=%s" % _minimum_pyarrow_version,
|
|
200
121
|
"grpcio>=%s" % _minimum_grpc_version,
|
|
201
122
|
"grpcio-status>=%s" % _minimum_grpc_version,
|
|
202
123
|
"googleapis-common-protos>=%s" % _minimum_googleapis_common_protos_version,
|
|
203
124
|
"numpy>=%s" % _minimum_numpy_version,
|
|
125
|
+
"pyyaml>=%s" % _minimum_pyyaml_version,
|
|
204
126
|
],
|
|
205
127
|
python_requires=">=3.9",
|
|
206
128
|
classifiers=[
|
|
@@ -210,6 +132,7 @@ try:
|
|
|
210
132
|
"Programming Language :: Python :: 3.10",
|
|
211
133
|
"Programming Language :: Python :: 3.11",
|
|
212
134
|
"Programming Language :: Python :: 3.12",
|
|
135
|
+
"Programming Language :: Python :: 3.13",
|
|
213
136
|
"Programming Language :: Python :: Implementation :: CPython",
|
|
214
137
|
"Programming Language :: Python :: Implementation :: PyPy",
|
|
215
138
|
"Typing :: Typed",
|
|
@@ -217,6 +140,8 @@ try:
|
|
|
217
140
|
)
|
|
218
141
|
finally:
|
|
219
142
|
if in_spark:
|
|
143
|
+
move("pyspark.back", "pyspark")
|
|
220
144
|
move("lib.back", "lib")
|
|
221
145
|
os.remove("setup.py")
|
|
222
146
|
os.remove("setup.cfg")
|
|
147
|
+
rmtree("pyspark_connect")
|
|
@@ -1,156 +0,0 @@
|
|
|
1
|
-
#
|
|
2
|
-
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
3
|
-
# contributor license agreements. See the NOTICE file distributed with
|
|
4
|
-
# this work for additional information regarding copyright ownership.
|
|
5
|
-
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
6
|
-
# (the "License"); you may not use this file except in compliance with
|
|
7
|
-
# the License. You may obtain a copy of the License at
|
|
8
|
-
#
|
|
9
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
-
#
|
|
11
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
12
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
-
# See the License for the specific language governing permissions and
|
|
15
|
-
# limitations under the License.
|
|
16
|
-
#
|
|
17
|
-
|
|
18
|
-
"""
|
|
19
|
-
PySpark is the Python API for Spark.
|
|
20
|
-
|
|
21
|
-
Public classes:
|
|
22
|
-
|
|
23
|
-
- :class:`SparkContext`:
|
|
24
|
-
Main entry point for Spark functionality.
|
|
25
|
-
- :class:`RDD`:
|
|
26
|
-
A Resilient Distributed Dataset (RDD), the basic abstraction in Spark.
|
|
27
|
-
- :class:`Broadcast`:
|
|
28
|
-
A broadcast variable that gets reused across tasks.
|
|
29
|
-
- :class:`Accumulator`:
|
|
30
|
-
An "add-only" shared variable that tasks can only add values to.
|
|
31
|
-
- :class:`SparkConf`:
|
|
32
|
-
For configuring Spark.
|
|
33
|
-
- :class:`SparkFiles`:
|
|
34
|
-
Access files shipped with jobs.
|
|
35
|
-
- :class:`StorageLevel`:
|
|
36
|
-
Finer-grained cache persistence levels.
|
|
37
|
-
- :class:`TaskContext`:
|
|
38
|
-
Information about the current running task, available on the workers and experimental.
|
|
39
|
-
- :class:`RDDBarrier`:
|
|
40
|
-
Wraps an RDD under a barrier stage for barrier execution.
|
|
41
|
-
- :class:`BarrierTaskContext`:
|
|
42
|
-
A :class:`TaskContext` that provides extra info and tooling for barrier execution.
|
|
43
|
-
- :class:`BarrierTaskInfo`:
|
|
44
|
-
Information about a barrier task.
|
|
45
|
-
- :class:`InheritableThread`:
|
|
46
|
-
A inheritable thread to use in Spark when the pinned thread mode is on.
|
|
47
|
-
"""
|
|
48
|
-
|
|
49
|
-
import sys
|
|
50
|
-
from functools import wraps
|
|
51
|
-
from typing import cast, Any, Callable, TypeVar, Union
|
|
52
|
-
|
|
53
|
-
from pyspark.util import is_remote_only
|
|
54
|
-
|
|
55
|
-
if not is_remote_only():
|
|
56
|
-
from pyspark.core.rdd import RDD, RDDBarrier
|
|
57
|
-
from pyspark.core.files import SparkFiles
|
|
58
|
-
from pyspark.core.status import StatusTracker, SparkJobInfo, SparkStageInfo
|
|
59
|
-
from pyspark.core.broadcast import Broadcast
|
|
60
|
-
from pyspark.core import rdd, files, status, broadcast
|
|
61
|
-
|
|
62
|
-
# for backward compatibility references.
|
|
63
|
-
sys.modules["pyspark.rdd"] = rdd
|
|
64
|
-
sys.modules["pyspark.files"] = files
|
|
65
|
-
sys.modules["pyspark.status"] = status
|
|
66
|
-
sys.modules["pyspark.broadcast"] = broadcast
|
|
67
|
-
|
|
68
|
-
from pyspark.conf import SparkConf
|
|
69
|
-
from pyspark.util import InheritableThread, inheritable_thread_target
|
|
70
|
-
from pyspark.storagelevel import StorageLevel
|
|
71
|
-
from pyspark.accumulators import Accumulator, AccumulatorParam
|
|
72
|
-
from pyspark.serializers import MarshalSerializer, CPickleSerializer
|
|
73
|
-
from pyspark.taskcontext import TaskContext, BarrierTaskContext, BarrierTaskInfo
|
|
74
|
-
from pyspark.profiler import Profiler, BasicProfiler
|
|
75
|
-
from pyspark.version import __version__
|
|
76
|
-
from pyspark._globals import _NoValue # noqa: F401
|
|
77
|
-
|
|
78
|
-
_F = TypeVar("_F", bound=Callable)
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
def since(version: Union[str, float]) -> Callable[[_F], _F]:
|
|
82
|
-
"""
|
|
83
|
-
A decorator that annotates a function to append the version of Spark the function was added.
|
|
84
|
-
"""
|
|
85
|
-
import re
|
|
86
|
-
|
|
87
|
-
indent_p = re.compile(r"\n( +)")
|
|
88
|
-
|
|
89
|
-
def deco(f: _F) -> _F:
|
|
90
|
-
assert f.__doc__ is not None
|
|
91
|
-
|
|
92
|
-
indents = indent_p.findall(f.__doc__)
|
|
93
|
-
indent = " " * (min(len(m) for m in indents) if indents else 0)
|
|
94
|
-
f.__doc__ = f.__doc__.rstrip() + "\n\n%s.. versionadded:: %s" % (indent, version)
|
|
95
|
-
return f
|
|
96
|
-
|
|
97
|
-
return deco
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
def keyword_only(func: _F) -> _F:
|
|
101
|
-
"""
|
|
102
|
-
A decorator that forces keyword arguments in the wrapped method
|
|
103
|
-
and saves actual input keyword arguments in `_input_kwargs`.
|
|
104
|
-
|
|
105
|
-
Notes
|
|
106
|
-
-----
|
|
107
|
-
Should only be used to wrap a method where first arg is `self`
|
|
108
|
-
"""
|
|
109
|
-
|
|
110
|
-
@wraps(func)
|
|
111
|
-
def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any:
|
|
112
|
-
if len(args) > 0:
|
|
113
|
-
raise TypeError("Method %s forces keyword arguments." % func.__name__)
|
|
114
|
-
self._input_kwargs = kwargs
|
|
115
|
-
return func(self, **kwargs)
|
|
116
|
-
|
|
117
|
-
return cast(_F, wrapper)
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
# To avoid circular dependencies
|
|
121
|
-
if not is_remote_only():
|
|
122
|
-
from pyspark.core.context import SparkContext
|
|
123
|
-
from pyspark.core import context
|
|
124
|
-
|
|
125
|
-
# for backward compatibility references.
|
|
126
|
-
sys.modules["pyspark.context"] = context
|
|
127
|
-
|
|
128
|
-
# for back compatibility
|
|
129
|
-
from pyspark.sql import SQLContext, HiveContext # noqa: F401
|
|
130
|
-
|
|
131
|
-
from pyspark.sql import Row # noqa: F401
|
|
132
|
-
|
|
133
|
-
__all__ = [
|
|
134
|
-
"SparkConf",
|
|
135
|
-
"SparkContext",
|
|
136
|
-
"SparkFiles",
|
|
137
|
-
"RDD",
|
|
138
|
-
"StorageLevel",
|
|
139
|
-
"Broadcast",
|
|
140
|
-
"Accumulator",
|
|
141
|
-
"AccumulatorParam",
|
|
142
|
-
"MarshalSerializer",
|
|
143
|
-
"CPickleSerializer",
|
|
144
|
-
"StatusTracker",
|
|
145
|
-
"SparkJobInfo",
|
|
146
|
-
"SparkStageInfo",
|
|
147
|
-
"Profiler",
|
|
148
|
-
"BasicProfiler",
|
|
149
|
-
"TaskContext",
|
|
150
|
-
"RDDBarrier",
|
|
151
|
-
"BarrierTaskContext",
|
|
152
|
-
"BarrierTaskInfo",
|
|
153
|
-
"InheritableThread",
|
|
154
|
-
"inheritable_thread_target",
|
|
155
|
-
"__version__",
|
|
156
|
-
]
|
|
@@ -1,71 +0,0 @@
|
|
|
1
|
-
#
|
|
2
|
-
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
3
|
-
# contributor license agreements. See the NOTICE file distributed with
|
|
4
|
-
# this work for additional information regarding copyright ownership.
|
|
5
|
-
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
6
|
-
# (the "License"); you may not use this file except in compliance with
|
|
7
|
-
# the License. You may obtain a copy of the License at
|
|
8
|
-
#
|
|
9
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
-
#
|
|
11
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
12
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
-
# See the License for the specific language governing permissions and
|
|
15
|
-
# limitations under the License.
|
|
16
|
-
#
|
|
17
|
-
|
|
18
|
-
"""
|
|
19
|
-
Module defining global singleton classes.
|
|
20
|
-
|
|
21
|
-
This module raises a RuntimeError if an attempt to reload it is made. In that
|
|
22
|
-
way the identities of the classes defined here are fixed and will remain so
|
|
23
|
-
even if pyspark itself is reloaded. In particular, a function like the following
|
|
24
|
-
will still work correctly after pyspark is reloaded:
|
|
25
|
-
|
|
26
|
-
def foo(arg=pyspark._NoValue):
|
|
27
|
-
if arg is pyspark._NoValue:
|
|
28
|
-
...
|
|
29
|
-
|
|
30
|
-
See gh-7844 for a discussion of the reload problem that motivated this module.
|
|
31
|
-
|
|
32
|
-
Note that this approach is taken after from NumPy.
|
|
33
|
-
"""
|
|
34
|
-
|
|
35
|
-
__ALL__ = ["_NoValue"]
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
# Disallow reloading this module so as to preserve the identities of the
|
|
39
|
-
# classes defined here.
|
|
40
|
-
if "_is_loaded" in globals():
|
|
41
|
-
raise RuntimeError("Reloading pyspark._globals is not allowed")
|
|
42
|
-
_is_loaded = True
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
class _NoValueType:
|
|
46
|
-
"""Special keyword value.
|
|
47
|
-
|
|
48
|
-
The instance of this class may be used as the default value assigned to a
|
|
49
|
-
deprecated keyword in order to check if it has been given a user defined
|
|
50
|
-
value.
|
|
51
|
-
|
|
52
|
-
This class was copied from NumPy.
|
|
53
|
-
"""
|
|
54
|
-
|
|
55
|
-
__instance = None
|
|
56
|
-
|
|
57
|
-
def __new__(cls):
|
|
58
|
-
# ensure that only one instance exists
|
|
59
|
-
if not cls.__instance:
|
|
60
|
-
cls.__instance = super(_NoValueType, cls).__new__(cls)
|
|
61
|
-
return cls.__instance
|
|
62
|
-
|
|
63
|
-
# needed for python 2 to preserve identity through a pickle
|
|
64
|
-
def __reduce__(self):
|
|
65
|
-
return (self.__class__, ())
|
|
66
|
-
|
|
67
|
-
def __repr__(self):
|
|
68
|
-
return "<no value>"
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
_NoValue = _NoValueType()
|
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
#
|
|
2
|
-
# Licensed to the Apache Software Foundation (ASF) under one
|
|
3
|
-
# or more contributor license agreements. See the NOTICE file
|
|
4
|
-
# distributed with this work for additional information
|
|
5
|
-
# regarding copyright ownership. The ASF licenses this file
|
|
6
|
-
# to you under the Apache License, Version 2.0 (the
|
|
7
|
-
# "License"); you may not use this file except in compliance
|
|
8
|
-
# with the License. You may obtain a copy of the License at
|
|
9
|
-
#
|
|
10
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
-
#
|
|
12
|
-
# Unless required by applicable law or agreed to in writing,
|
|
13
|
-
# software distributed under the License is distributed on an
|
|
14
|
-
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
15
|
-
# KIND, either express or implied. See the License for the
|
|
16
|
-
# specific language governing permissions and limitations
|
|
17
|
-
# under the License.
|
|
18
|
-
|
|
19
|
-
from typing import Callable, Iterable, Sized, TypeVar, Union
|
|
20
|
-
from typing_extensions import Literal, Protocol
|
|
21
|
-
|
|
22
|
-
from numpy import int32, int64, float32, float64, ndarray
|
|
23
|
-
|
|
24
|
-
F = TypeVar("F", bound=Callable)
|
|
25
|
-
T_co = TypeVar("T_co", covariant=True)
|
|
26
|
-
|
|
27
|
-
PrimitiveType = Union[bool, float, int, str]
|
|
28
|
-
|
|
29
|
-
NonUDFType = Literal[0]
|
|
30
|
-
|
|
31
|
-
class SupportsIAdd(Protocol):
|
|
32
|
-
def __iadd__(self, other: SupportsIAdd) -> SupportsIAdd: ...
|
|
33
|
-
|
|
34
|
-
class SupportsOrdering(Protocol):
|
|
35
|
-
def __lt__(self, other: SupportsOrdering) -> bool: ...
|
|
36
|
-
|
|
37
|
-
class SizedIterable(Protocol, Sized, Iterable[T_co]): ...
|
|
38
|
-
|
|
39
|
-
S = TypeVar("S", bound=SupportsOrdering)
|
|
40
|
-
|
|
41
|
-
NumberOrArray = TypeVar(
|
|
42
|
-
"NumberOrArray", float, int, complex, int32, int64, float32, float64, ndarray
|
|
43
|
-
)
|