sws-spark-dissemination-helper 0.0.113__tar.gz → 0.0.115__tar.gz

{sws_spark_dissemination_helper-0.0.113 → sws_spark_dissemination_helper-0.0.115}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.113
+Version: 0.0.115
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://bitbucket.org/cioapps/sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>
@@ -49,7 +49,7 @@ Requires-Dist: pytz==2025.1
 Requires-Dist: requests==2.32.3
 Requires-Dist: s3transfer==0.11.2
 Requires-Dist: six==1.17.0
-Requires-Dist: sws-api-client==1.4.5
+Requires-Dist: sws-api-client>=1.5.0
 Requires-Dist: typing-extensions==4.12.2
 Requires-Dist: tzdata==2025.1
 Requires-Dist: urllib3==1.26.20

{sws_spark_dissemination_helper-0.0.113 → sws_spark_dissemination_helper-0.0.115}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.113"
+version = "0.0.115"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3==1.36.18",
@@ -25,7 +25,7 @@ dependencies = [
     "requests==2.32.3",
     "s3transfer==0.11.2",
     "six==1.17.0",
-    "sws_api_client==1.4.5",
+    "sws_api_client>=1.5.0",
     "typing_extensions==4.12.2",
     "tzdata==2025.1",
     "urllib3==1.26.20"

{sws_spark_dissemination_helper-0.0.113 → sws_spark_dissemination_helper-0.0.115}/requirements.txt

@@ -17,7 +17,7 @@ pytz==2025.1
 requests==2.32.3
 s3transfer==0.11.2
 six==1.17.0
-sws_api_client==1.4.5
+sws_api_client==1.5.0
 typing_extensions==4.12.2
 tzdata==2025.1
 urllib3==1.26.20

{sws_spark_dissemination_helper-0.0.113 → sws_spark_dissemination_helper-0.0.115}/src/sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py

@@ -417,11 +417,13 @@ class SWSBronzeIcebergSparkHelper:
             description="Bronze table containing all the raw data imported from the SWS and denormalized",
             layer=TableLayer.BRONZE,
             private=True,
+            debug=True,
             type=TableType.ICEBERG,
             database=IcebergDatabases.BRONZE_DATABASE,
             table=self.iceberg_tables.BRONZE.table,
             path=self.iceberg_tables.BRONZE.path,
             structure={"columns": self.df_bronze.schema.jsonValue()["fields"]},
+            pinned_columns=[*self.dim_columns_w_time, "value", *self.flag_columns],
         )
         tag = tags.add_dissemination_table(
             self.dataset_id, self.tag_name, new_iceberg_table
@@ -434,6 +436,7 @@ class SWSBronzeIcebergSparkHelper:
             description="Bronze table containing all the raw data imported from the SWS and denormalized cached in csv",
             layer=TableLayer.BRONZE,
             private=True,
+            debug=True,
             type=TableType.CSV,
             path=self.iceberg_tables.BRONZE.csv_path,
             structure={"columns": self.df_bronze.schema.jsonValue()["fields"]},
@@ -502,6 +505,7 @@ class SWSBronzeIcebergSparkHelper:
             structure={
                 "columns": self.disseminated_tag_df.schema.jsonValue()["fields"]
             },
+            pinned_columns=[*self.dim_columns_w_time, "value", *self.flag_columns],
         )
         tag = tags.add_dissemination_table(
             self.dataset_id, self.tag_name, new_iceberg_table

{sws_spark_dissemination_helper-0.0.113 → sws_spark_dissemination_helper-0.0.115}/src/sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py

@@ -138,9 +138,7 @@ class SWSGoldIcebergSparkHelper:
 
         return df
 
-    def write_gold_sws_data_to_iceberg_and_csv(
-        self, df: DataFrame
-    ) -> DataFrame:
+    def write_gold_sws_data_to_iceberg_and_csv(self, df: DataFrame) -> DataFrame:
         df.writeTo(self.iceberg_tables.GOLD_SWS.iceberg_id).createOrReplace()
 
         logging.info(
@@ -433,6 +431,7 @@ class SWSGoldIcebergSparkHelper:
             description="Gold table containing all the cleaned data in SDMX compatible format, ready to be mapped using FMR",
             layer=TableLayer.GOLD,
             private=True,
+            debug=True,
             type=TableType.ICEBERG,
             database=IcebergDatabases.GOLD_DATABASE,
             table=self.iceberg_tables.GOLD_PRE_SDMX.table,
@@ -454,6 +453,7 @@ class SWSGoldIcebergSparkHelper:
             description="Gold table containing all the cleaned data in SDMX compatible format, ready to be mapped using FMR and cached in csv",
             layer=TableLayer.GOLD,
             private=True,
+            debug=True,
             type=TableType.CSV,
             path=self.iceberg_tables.GOLD_PRE_SDMX.csv_path,
             structure={"columns": df.schema.jsonValue()["fields"]},
@@ -469,9 +469,7 @@ class SWSGoldIcebergSparkHelper:
 
         return df
 
-    def write_gold_sws_dissemination_tag(
-        self, df: DataFrame, tags: Tags
-    ) -> DataFrame:
+    def write_gold_sws_dissemination_tag(self, df: DataFrame, tags: Tags) -> DataFrame:
         # Get or create a new tag
         tag = get_or_create_tag(tags, self.dataset_id, self.tag_name, self.tag_name)
         logging.debug(f"Tag: {tag}")

{sws_spark_dissemination_helper-0.0.113 → sws_spark_dissemination_helper-0.0.115}/src/sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py

@@ -656,6 +656,13 @@ class SWSSilverIcebergSparkHelper:
             type=TableType.CSV,
             path=self.iceberg_tables.SILVER.csv_path,
             structure={"columns": df.schema.jsonValue()["fields"]},
+            pinned_columns=[
+                *self.dim_columns_w_time,
+                "value",
+                *self.flag_columns,
+                "diss_flag",
+                "note",
+            ],
         )
         tag = upsert_disseminated_table(
             sws_tags=tags,