dataproc-spark-connect 0.7.0__py2.py3-none-any.whl → 0.7.2__py2.py3-none-any.whl

{dataproc_spark_connect-0.7.0.dist-info → dataproc_spark_connect-0.7.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dataproc-spark-connect
-Version: 0.7.0
+Version: 0.7.2
 Summary: Dataproc client library for Spark Connect
 Home-page: https://github.com/GoogleCloudDataproc/dataproc-spark-connect-python
 Author: Google LLC
@@ -11,7 +11,7 @@ Requires-Dist: google-cloud-dataproc>=5.18
 Requires-Dist: packaging>=20.0
 Requires-Dist: pyspark[connect]>=3.5
 Requires-Dist: tqdm>=4.67
-Requires-Dist: websockets>=15.0
+Requires-Dist: websockets>=14.0
 
 # Dataproc Spark Connect Client
 

{dataproc_spark_connect-0.7.0.dist-info → dataproc_spark_connect-0.7.2.dist-info}/RECORD

@@ -1,12 +1,12 @@
 google/cloud/dataproc_spark_connect/__init__.py,sha256=dIqHNWVWWrSuRf26x11kX5e9yMKSHCtmI_GBj1-FDdE,1101
 google/cloud/dataproc_spark_connect/exceptions.py,sha256=ilGyHD5M_yBQ3IC58-Y5miRGIQVJsLaNKvEGcHuk_BE,969
 google/cloud/dataproc_spark_connect/pypi_artifacts.py,sha256=gd-VMwiVP-EJuPp9Vf9Shx8pqps3oSKp0hBcSSZQS-A,1575
-google/cloud/dataproc_spark_connect/session.py,sha256=98Zrn0Vyl2ajcF5hltdSp8LgYTOzDa-eqeYxxmZVKds,26398
+google/cloud/dataproc_spark_connect/session.py,sha256=7CgRmYVp8M4oVosvxz08jp72BU4ZtAKvxjtd0101GHs,24952
 google/cloud/dataproc_spark_connect/client/__init__.py,sha256=6hCNSsgYlie6GuVpc5gjFsPnyeMTScTpXSPYqp1fplY,615
 google/cloud/dataproc_spark_connect/client/core.py,sha256=m3oXTKBm3sBy6jhDu9GRecrxLb5CdEM53SgMlnJb6ag,4616
 google/cloud/dataproc_spark_connect/client/proxy.py,sha256=qUZXvVY1yn934vE6nlO495XUZ53AUx9O74a9ozkGI9U,8976
-dataproc_spark_connect-0.7.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-dataproc_spark_connect-0.7.0.dist-info/METADATA,sha256=fFJLyzjo3CKLx1d18U4i1csJEPgWxoAjok4qFghtOyE,3328
-dataproc_spark_connect-0.7.0.dist-info/WHEEL,sha256=OpXWERl2xLPRHTvd2ZXo_iluPEQd8uSbYkJ53NAER_Y,109
-dataproc_spark_connect-0.7.0.dist-info/top_level.txt,sha256=_1QvSJIhFAGfxb79D6DhB7SUw2X6T4rwnz_LLrbcD3c,7
-dataproc_spark_connect-0.7.0.dist-info/RECORD,,
+dataproc_spark_connect-0.7.2.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+dataproc_spark_connect-0.7.2.dist-info/METADATA,sha256=EtPp009uOCvR1I3tiT-x1FJ4bWpPHhEqZxvJO4rfwKk,3328
+dataproc_spark_connect-0.7.2.dist-info/WHEEL,sha256=OpXWERl2xLPRHTvd2ZXo_iluPEQd8uSbYkJ53NAER_Y,109
+dataproc_spark_connect-0.7.2.dist-info/top_level.txt,sha256=_1QvSJIhFAGfxb79D6DhB7SUw2X6T4rwnz_LLrbcD3c,7
+dataproc_spark_connect-0.7.2.dist-info/RECORD,,

google/cloud/dataproc_spark_connect/session.py

@@ -77,13 +77,6 @@ class DataprocSparkSession(SparkSession):
 
     class Builder(SparkSession.Builder):
 
-        _dataproc_runtime_to_spark_version = {
-            "1.2": "3.5",
-            "2.2": "3.5",
-            "2.3": "3.5",
-            "3.0": "4.0",
-        }
-
         _session_static_configs = [
             "spark.executor.cores",
             "spark.executor.memoryOverhead",
@@ -188,8 +181,6 @@ class DataprocSparkSession(SparkSession):
 
                 dataproc_config: Session = self._get_dataproc_config()
 
-                self._validate_version(dataproc_config)
-
                 session_id = self.generate_dataproc_session_id()
                 dataproc_config.name = f"projects/{self._project_id}/locations/{self._region}/sessions/{session_id}"
                 logger.debug(
@@ -304,6 +295,8 @@ class DataprocSparkSession(SparkSession):
                     raise RuntimeError(
                         f"Error while creating Dataproc Session"
                     ) from e
+                finally:
+                    stop_create_session_pbar = True
 
                 logger.debug(
                     f"Dataproc Session created: {session_id} in {int(time.time() - s8s_creation_start_time)} seconds"
@@ -417,36 +410,6 @@ class DataprocSparkSession(SparkSession):
                 ]
             return dataproc_config
 
-        def _validate_version(self, dataproc_config):
-            trim_version = lambda v: ".".join(v.split(".")[:2])
-
-            version = dataproc_config.runtime_config.version
-            if (
-                trim_version(version)
-                not in self._dataproc_runtime_to_spark_version
-            ):
-                raise ValueError(
-                    f"Specified {version} Dataproc Spark runtime version is not supported. "
-                    f"Supported runtime versions: {self._dataproc_runtime_to_spark_version.keys()}"
-                )
-
-            server_version = self._dataproc_runtime_to_spark_version[
-                trim_version(version)
-            ]
-
-            import importlib.metadata
-
-            dataproc_connect_version = importlib.metadata.version(
-                "dataproc-spark-connect"
-            )
-            client_version = importlib.metadata.version("pyspark")
-            if trim_version(client_version) != trim_version(server_version):
-                print(
-                    f"Spark Connect client and server use different versions:\n"
-                    f"- Dataproc Spark Connect client {dataproc_connect_version} (PySpark {client_version})\n"
-                    f"- Dataproc Spark runtime {version} (Spark {server_version})"
-                )
-
         @staticmethod
         def generate_dataproc_session_id():
             timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")