pyspark-client 4.2.0.dev4__tar.gz → 4.2.0.dev5__tar.gz

{pyspark_client-4.2.0.dev4/pyspark_client.egg-info → pyspark_client-4.2.0.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyspark-client
-Version: 4.2.0.dev4
+Version: 4.2.0.dev5
 Summary: Python Spark Connect client for Apache Spark
 Home-page: https://github.com/apache/spark/tree/master/python
 Author: Spark Developers

{pyspark_client-4.2.0.dev4 → pyspark_client-4.2.0.dev5}/pyspark/errors/error-conditions.json

@@ -808,14 +808,14 @@
       "Number of columns of the returned data doesn't match specified schema. Expected: <expected> Actual: <actual>"
     ]
   },
-  "RESULT_ROWS_MISMATCH": {
+  "RESULT_COLUMN_TYPES_MISMATCH": {
     "message": [
-      "The number of output rows (<output_length>) must match the number of input rows (<input_length>)."
+      "Column types of the returned data do not match specified schema. Mismatch: <mismatch>."
     ]
   },
-  "RESULT_TYPE_MISMATCH_FOR_ARROW_UDF": {
+  "RESULT_ROWS_MISMATCH": {
     "message": [
-      "Columns do not match in their data type: <mismatch>."
+      "The number of output rows (<output_length>) must match the number of input rows (<input_length>)."
     ]
   },
   "REUSE_OBSERVATION": {
@@ -823,11 +823,6 @@
       "An Observation can be used with a DataFrame only once."
     ]
   },
-  "SCHEMA_MISMATCH_FOR_ARROW_PYTHON_UDF": {
-    "message": [
-      "Result vector from <udf_type> was not the required length: expected <expected>, got <actual>."
-    ]
-  },
   "SCHEMA_MISMATCH_FOR_PANDAS_UDF": {
     "message": [
       "Result vector from <udf_type> was not the required length: expected <expected>, got <actual>."

{pyspark_client-4.2.0.dev4 → pyspark_client-4.2.0.dev5}/pyspark/install.py

@@ -68,7 +68,7 @@ def checked_versions(
         fully-qualified versions of Spark, Hadoop and Hive in a tuple.
         For example, spark-3.2.0, hadoop3 and hive2.3.
     """
-    if re.match("^[0-9]+\\.[0-9]+\\.[0-9]+$", spark_version):
+    if re.match("^[0-9]+\\.[0-9]+\\.[0-9]+(?:\\.dev[0-9]+)?$", spark_version):
         spark_version = "spark-%s" % spark_version
     if not spark_version.startswith("spark-"):
         raise RuntimeError(
@@ -106,7 +106,9 @@ def convert_old_hadoop_version(spark_version: str, hadoop_version: str) -> str:
         "without": "without",
         "without-hadoop": "without-hadoop",
     }
-    spark_version_parts = re.search("^spark-([0-9]+)\\.([0-9]+)\\.[0-9]+$", spark_version)
+    spark_version_parts = re.search(
+        "^spark-([0-9]+)\\.([0-9]+)\\.[0-9]+(?:\\.dev[0-9]+)?$", spark_version
+    )
     assert spark_version_parts is not None
     spark_major_version = int(spark_version_parts.group(1))
     spark_minor_version = int(spark_version_parts.group(2))

{pyspark_client-4.2.0.dev4 → pyspark_client-4.2.0.dev5}/pyspark/memory_profiler_ext.py

@@ -123,7 +123,7 @@ def _init_module() -> None:
                 measures = self[code]
                 if not measures:
                     continue  # skip if no measurement
-                line_iterator = ((line, measures[line]) for line in measures.keys())
+                line_iterator = ((line, measures[line]) for line in measures)
                 yield (filename, line_iterator)
 
     class UDFLineProfiler(LineProfiler):

{pyspark_client-4.2.0.dev4 → pyspark_client-4.2.0.dev5}/pyspark/pandas/config.py

@@ -508,9 +508,7 @@ class DictWrapper:
             prefix += "."
         canonical_key = prefix + key
 
-        candidates = [
-            k for k in d.keys() if all(x in k.split(".") for x in canonical_key.split("."))
-        ]
+        candidates = [k for k in d if all(x in k.split(".") for x in canonical_key.split("."))]
         if len(candidates) == 1 and candidates[0] == canonical_key:
             set_option(canonical_key, val)
         else:
@@ -527,9 +525,7 @@ class DictWrapper:
             prefix += "."
         canonical_key = prefix + key
 
-        candidates = [
-            k for k in d.keys() if all(x in k.split(".") for x in canonical_key.split("."))
-        ]
+        candidates = [k for k in d if all(x in k.split(".") for x in canonical_key.split("."))]
         if len(candidates) == 1 and candidates[0] == canonical_key:
             return get_option(canonical_key)
         elif len(candidates) == 0:
@@ -549,7 +545,7 @@ class DictWrapper:
             candidates = d.keys()
             offset = 0
         else:
-            candidates = [k for k in d.keys() if all(x in k.split(".") for x in prefix.split("."))]
+            candidates = [k for k in d if all(x in k.split(".") for x in prefix.split("."))]
             offset = len(prefix) + 1  # prefix (e.g. "compute.") to trim.
         return [c[offset:] for c in candidates]
 

{pyspark_client-4.2.0.dev4 → pyspark_client-4.2.0.dev5}/pyspark/pandas/frame.py

@@ -7072,7 +7072,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
                 index_map[colname] = None
             internal = InternalFrame(
                 spark_frame=sdf,
-                index_spark_columns=[scol_for(sdf, col) for col in index_map.keys()],
+                index_spark_columns=[scol_for(sdf, col) for col in index_map],
                 index_names=list(index_map.values()),
                 column_label_names=[columns],
             )
@@ -7988,7 +7988,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
             first puts NaNs at the beginning, last puts NaNs at the end. Not implemented for
             MultiIndex.
         ignore_index : bool, default False
-            If True, the resulting axis will be labeled 0, 1, …, n - 1.
+            If True, the resulting axis will be labeled 0, 1, ..., n - 1.
 
             .. versionadded:: 3.4.0
 
@@ -9646,7 +9646,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
         random_state : int, optional
             Seed for the random number generator (if int).
         ignore_index : bool, default False
-            If True, the resulting index will be labeled 0, 1, …, n - 1.
+            If True, the resulting index will be labeled 0, 1, ..., n - 1.
 
             .. versionadded:: 3.4.0
 
@@ -9778,7 +9778,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
         applied = []
         if is_dict_like(dtype):
             dtype_dict = cast(Dict[Name, Union[str, Dtype]], dtype)
-            for col_name in dtype_dict.keys():
+            for col_name in dtype_dict:
                 if col_name not in self.columns:
                     raise KeyError(
                         "Only a column name can be used for the key in a dtype mappings argument."
@@ -10249,7 +10249,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
         inplace : boolean, default False
             Whether to drop duplicates in place or to return a copy.
         ignore_index : boolean, default False
-            If True, the resulting axis will be labeled 0, 1, …, n - 1.
+            If True, the resulting axis will be labeled 0, 1, ..., n - 1.
 
         Returns
         -------
@@ -13351,7 +13351,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
         column : str or tuple
             Column to explode.
         ignore_index : bool, default False
-            If True, the resulting index will be labeled 0, 1, …, n - 1.
+            If True, the resulting index will be labeled 0, 1, ..., n - 1.
 
         Returns
         -------

{pyspark_client-4.2.0.dev4 → pyspark_client-4.2.0.dev5}/pyspark/pandas/indexes/base.py

@@ -872,7 +872,7 @@ class Index(IndexOpsMixin):
         with ps.option_context("compute.default_index_type", "distributed"):
             # The attached index caused by `reset_index` below is used for sorting only,
             # and it will be dropped soon,
-            # so we enforce “distributed” default index type
+            # so we enforce "distributed" default index type
             psser = self.to_series().reset_index(drop=True)
         return Index(psser.drop_duplicates(keep=keep).sort_index())
 

{pyspark_client-4.2.0.dev4 → pyspark_client-4.2.0.dev5}/pyspark/pandas/indexes/datetimes.py

@@ -806,7 +806,7 @@ class DatetimeIndex(Index):
         psdf = psdf.pandas_on_spark.attach_id_column("distributed-sequence", id_column_name)
         with ps.option_context("compute.default_index_type", "distributed"):
             # The attached index in the statement below will be dropped soon,
-            # so we enforce “distributed” default index type
+            # so we enforce "distributed" default index type
             psdf = psdf.pandas_on_spark.apply_batch(pandas_between_time)
         return ps.Index(first_series(psdf).rename(self.name))
 
@@ -851,7 +851,7 @@ class DatetimeIndex(Index):
         psdf = psdf.pandas_on_spark.attach_id_column("distributed-sequence", id_column_name)
         with ps.option_context("compute.default_index_type", "distributed"):
             # The attached index in the statement below will be dropped soon,
-            # so we enforce “distributed” default index type
+            # so we enforce "distributed" default index type
             psdf = psdf.pandas_on_spark.apply_batch(pandas_at_time)
         return ps.Index(first_series(psdf).rename(self.name))
 

{pyspark_client-4.2.0.dev4 → pyspark_client-4.2.0.dev5}/pyspark/pandas/indexes/multi.py

@@ -961,7 +961,7 @@ class MultiIndex(Index):
         with ps.option_context("compute.default_index_type", "distributed"):
             # The attached index caused by `reset_index` below is used for sorting only,
             # and it will be dropped soon,
-            # so we enforce “distributed” default index type
+            # so we enforce "distributed" default index type
             psdf = self.to_frame().reset_index(drop=True)
         return ps.MultiIndex.from_frame(psdf.drop_duplicates(keep=keep).sort_index())
 

{pyspark_client-4.2.0.dev4 → pyspark_client-4.2.0.dev5}/pyspark/pandas/namespace.py

@@ -1784,7 +1784,7 @@ def to_datetime(
     if isinstance(arg, Series):
         return arg.pandas_on_spark.transform_batch(pandas_to_datetime)
     if isinstance(arg, DataFrame):
-        unit = {k: _unit_map[k.lower()] for k in arg.keys() if k.lower() in _unit_map}
+        unit = {k: _unit_map[k.lower()] for k in arg if k.lower() in _unit_map}
         unit_rev = {v: k for k, v in unit.items()}
         list_cols = [unit_rev["year"], unit_rev["month"], unit_rev["day"]]
         for u in ["h", "m", "s", "ms", "us"]:

{pyspark_client-4.2.0.dev4 → pyspark_client-4.2.0.dev5}/pyspark/pipelines/cli.py

@@ -165,7 +165,7 @@ def unpack_pipeline_spec(spec_data: Mapping[str, Any]) -> PipelineSpec:
         "libraries",
     }
     REQUIRED_FIELDS = ["name", "storage"]
-    for key in spec_data.keys():
+    for key in spec_data:
         if key not in ALLOWED_FIELDS:
             raise PySparkException(
                 errorClass="PIPELINE_SPEC_UNEXPECTED_FIELD", messageParameters={"field_name": key}