sws-spark-dissemination-helper 0.0.115__tar.gz → 0.0.117__tar.gz

{sws_spark_dissemination_helper-0.0.115 → sws_spark_dissemination_helper-0.0.117}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.115
+Version: 0.0.117
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://bitbucket.org/cioapps/sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.115 → sws_spark_dissemination_helper-0.0.117}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.115"
+version = "0.0.117"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3==1.36.18",

{sws_spark_dissemination_helper-0.0.115 → sws_spark_dissemination_helper-0.0.117}/src/sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py

@@ -176,6 +176,9 @@ class SWSSilverIcebergSparkHelper:
 
             original_col_order = df.columns
             cols_to_select = df.columns
+            logging.info("df.columns")
+            logging.info(f"{col_name}_start_date")
+            logging.info(f"{col_name}_end_date")
             cols_to_select.remove("note")
             cols_to_select.remove(f"{col_name}_start_date")
             cols_to_select.remove(f"{col_name}_end_date")
@@ -637,6 +640,13 @@ class SWSSilverIcebergSparkHelper:
             table=self.iceberg_tables.SILVER.table,
             path=self.iceberg_tables.SILVER.path,
             structure={"columns": df.schema.jsonValue()["fields"]},
+            pinned_columns=[
+                *self.dim_columns_w_time,
+                "value",
+                *self.flag_columns,
+                "diss_flag",
+                "note",
+            ],
         )
         tag = upsert_disseminated_table(
             sws_tags=tags,
@@ -656,13 +666,6 @@ class SWSSilverIcebergSparkHelper:
             type=TableType.CSV,
             path=self.iceberg_tables.SILVER.csv_path,
             structure={"columns": df.schema.jsonValue()["fields"]},
-            pinned_columns=[
-                *self.dim_columns_w_time,
-                "value",
-                *self.flag_columns,
-                "diss_flag",
-                "note",
-            ],
         )
         tag = upsert_disseminated_table(
             sws_tags=tags,