dataproc-spark-connect 0.7.3__py2.py3-none-any.whl → 0.7.5__py2.py3-none-any.whl

{dataproc_spark_connect-0.7.3.dist-info → dataproc_spark_connect-0.7.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dataproc-spark-connect
-Version: 0.7.3
+Version: 0.7.5
 Summary: Dataproc client library for Spark Connect
 Home-page: https://github.com/GoogleCloudDataproc/dataproc-spark-connect-python
 Author: Google LLC
@@ -9,7 +9,7 @@ License-File: LICENSE
 Requires-Dist: google-api-core>=2.19
 Requires-Dist: google-cloud-dataproc>=5.18
 Requires-Dist: packaging>=20.0
-Requires-Dist: pyspark[connect]>=3.5
+Requires-Dist: pyspark[connect]~=3.5.1
 Requires-Dist: tqdm>=4.67
 Requires-Dist: websockets>=14.0
 

{dataproc_spark_connect-0.7.3.dist-info → dataproc_spark_connect-0.7.5.dist-info}/RECORD

@@ -1,12 +1,12 @@
 google/cloud/dataproc_spark_connect/__init__.py,sha256=dIqHNWVWWrSuRf26x11kX5e9yMKSHCtmI_GBj1-FDdE,1101
 google/cloud/dataproc_spark_connect/exceptions.py,sha256=ilGyHD5M_yBQ3IC58-Y5miRGIQVJsLaNKvEGcHuk_BE,969
 google/cloud/dataproc_spark_connect/pypi_artifacts.py,sha256=gd-VMwiVP-EJuPp9Vf9Shx8pqps3oSKp0hBcSSZQS-A,1575
-google/cloud/dataproc_spark_connect/session.py,sha256=34hEgR57qi8vyM0_j749TCcpPeprJrEYXOzuAAGUrRg,25020
+google/cloud/dataproc_spark_connect/session.py,sha256=kMCZWmi_-ScJy9NO7NFrHaHDTXKxMwaCSDbdqGxEngk,26390
 google/cloud/dataproc_spark_connect/client/__init__.py,sha256=6hCNSsgYlie6GuVpc5gjFsPnyeMTScTpXSPYqp1fplY,615
 google/cloud/dataproc_spark_connect/client/core.py,sha256=m3oXTKBm3sBy6jhDu9GRecrxLb5CdEM53SgMlnJb6ag,4616
 google/cloud/dataproc_spark_connect/client/proxy.py,sha256=qUZXvVY1yn934vE6nlO495XUZ53AUx9O74a9ozkGI9U,8976
-dataproc_spark_connect-0.7.3.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-dataproc_spark_connect-0.7.3.dist-info/METADATA,sha256=pj2QInvZOehYc9RLMp0XUfz9nzA9rZ76PKaZDAx0-nA,3328
-dataproc_spark_connect-0.7.3.dist-info/WHEEL,sha256=OpXWERl2xLPRHTvd2ZXo_iluPEQd8uSbYkJ53NAER_Y,109
-dataproc_spark_connect-0.7.3.dist-info/top_level.txt,sha256=_1QvSJIhFAGfxb79D6DhB7SUw2X6T4rwnz_LLrbcD3c,7
-dataproc_spark_connect-0.7.3.dist-info/RECORD,,
+dataproc_spark_connect-0.7.5.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+dataproc_spark_connect-0.7.5.dist-info/METADATA,sha256=byc2dTo3PdkmbMZHyaJ9A-WVhSQTHn1ZAX_Jqz9jmd0,3330
+dataproc_spark_connect-0.7.5.dist-info/WHEEL,sha256=OpXWERl2xLPRHTvd2ZXo_iluPEQd8uSbYkJ53NAER_Y,109
+dataproc_spark_connect-0.7.5.dist-info/top_level.txt,sha256=_1QvSJIhFAGfxb79D6DhB7SUw2X6T4rwnz_LLrbcD3c,7
+dataproc_spark_connect-0.7.5.dist-info/RECORD,,

google/cloud/dataproc_spark_connect/session.py

@@ -408,6 +408,30 @@ class DataprocSparkSession(SparkSession):
                 dataproc_config.labels["colab-notebook-kernel-id"] = os.environ[
                     "COLAB_NOTEBOOK_KERNEL_ID"
                 ]
+            default_datasource = os.getenv(
+                "DATAPROC_SPARK_CONNECT_DEFAULT_DATASOURCE"
+            )
+            if (
+                default_datasource
+                and dataproc_config.runtime_config.version == "2.3"
+            ):
+                if default_datasource == "bigquery":
+                    bq_datasource_properties = {
+                        "spark.datasource.bigquery.viewsEnabled": "true",
+                        "spark.datasource.bigquery.writeMethod": "direct",
+                        "spark.sql.catalog.spark_catalog": "com.google.cloud.spark.bigquery.BigQuerySparkSessionCatalog",
+                        "spark.sql.legacy.createHiveTableByDefault": "false",
+                        "spark.sql.sources.default": "bigquery",
+                    }
+                    # Merge default configs with existing properties, user configs take precedence
+                    for k, v in bq_datasource_properties.items():
+                        if k not in dataproc_config.runtime_config.properties:
+                            dataproc_config.runtime_config.properties[k] = v
+                else:
+                    logger.warning(
+                        f"DATAPROC_SPARK_CONNECT_DEFAULT_DATASOURCE is set to an invalid value:"
+                        f" {default_datasource}. Supported value is 'bigquery'."
+                    )
             return dataproc_config
 
         @staticmethod