dataproc-spark-connect 0.7.3__tar.gz → 0.7.4__tar.gz

{dataproc_spark_connect-0.7.3 → dataproc_spark_connect-0.7.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dataproc-spark-connect
-Version: 0.7.3
+Version: 0.7.4
 Summary: Dataproc client library for Spark Connect
 Home-page: https://github.com/GoogleCloudDataproc/dataproc-spark-connect-python
 Author: Google LLC

{dataproc_spark_connect-0.7.3 → dataproc_spark_connect-0.7.4}/dataproc_spark_connect.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dataproc-spark-connect
-Version: 0.7.3
+Version: 0.7.4
 Summary: Dataproc client library for Spark Connect
 Home-page: https://github.com/GoogleCloudDataproc/dataproc-spark-connect-python
 Author: Google LLC

{dataproc_spark_connect-0.7.3 → dataproc_spark_connect-0.7.4}/google/cloud/dataproc_spark_connect/session.py

@@ -408,6 +408,30 @@ class DataprocSparkSession(SparkSession):
                 dataproc_config.labels["colab-notebook-kernel-id"] = os.environ[
                     "COLAB_NOTEBOOK_KERNEL_ID"
                 ]
+            default_datasource = os.getenv(
+                "DATAPROC_SPARK_CONNECT_DEFAULT_DATASOURCE"
+            )
+            if (
+                default_datasource
+                and dataproc_config.runtime_config.version == "2.3"
+            ):
+                if default_datasource == "bigquery":
+                    bq_datasource_properties = {
+                        "spark.datasource.bigquery.viewsEnabled": "true",
+                        "spark.datasource.bigquery.writeMethod": "direct",
+                        "spark.sql.catalog.spark_catalog": "com.google.cloud.spark.bigquery.BigQuerySparkSessionCatalog",
+                        "spark.sql.legacy.createHiveTableByDefault": "false",
+                        "spark.sql.sources.default": "bigquery",
+                    }
+                    # Merge default configs with existing properties, user configs take precedence
+                    for k, v in bq_datasource_properties.items():
+                        if k not in dataproc_config.runtime_config.properties:
+                            dataproc_config.runtime_config.properties[k] = v
+                else:
+                    logger.warning(
+                        f"DATAPROC_SPARK_CONNECT_DEFAULT_DATASOURCE is set to an invalid value:"
+                        f" {default_datasource}. Supported value is 'bigquery'."
+                    )
             return dataproc_config
 
         @staticmethod

{dataproc_spark_connect-0.7.3 → dataproc_spark_connect-0.7.4}/setup.py

@@ -20,7 +20,7 @@ long_description = (this_directory / "README.md").read_text()
 
 setup(
     name="dataproc-spark-connect",
-    version="0.7.3",
+    version="0.7.4",
     description="Dataproc client library for Spark Connect",
     long_description=long_description,
     author="Google LLC",