sws-spark-dissemination-helper 0.0.179__py3-none-any.whl → 0.0.181__py3-none-any.whl

sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py

@@ -88,10 +88,13 @@ class SWSGoldIcebergSparkHelper:
     def apply_diss_flag_filter(self, df: DataFrame) -> DataFrame:
         return df.filter(col("diss_flag"))
 
-    def keep_dim_val_attr_columns(self, df: DataFrame):
+    def keep_dim_val_attr_columns(
+        self, df: DataFrame, additional_columns: List[str] = []
+    ):
         cols_to_keep_sws = self.cols_to_keep_sws
-        if "note" in df.columns:
-            cols_to_keep_sws = cols_to_keep_sws + ["note"]
+        for additional_column in additional_columns:
+            if additional_column in df.columns:
+                cols_to_keep_sws = cols_to_keep_sws + [additional_column]
         if "unit_of_measure_symbol" in df.columns:
             cols_to_keep_sws = cols_to_keep_sws + ["unit_of_measure_symbol"]
         return df.select(*cols_to_keep_sws)
@@ -137,7 +140,7 @@ class SWSGoldIcebergSparkHelper:
             F.round(
                 F.col("value").cast("FLOAT") * F.pow(10, F.col("display_decimals")), 0
             )
-            / F.pow(10, F.col("dec")).cast("STRING"),
+            / F.pow(10, F.col("display_decimals")).cast("STRING"),
         )
 
         # F.round(
@@ -156,18 +159,26 @@ class SWSGoldIcebergSparkHelper:
             self.iceberg_tables.SILVER.iceberg_id
         )
 
-    def gen_gold_sws_disseminated_data(self) -> DataFrame:
+    def gen_gold_sws_disseminated_data(
+        self, additional_columns: List[str] = []
+    ) -> DataFrame:
         return (
             self.read_silver_data()
             .transform(self.apply_diss_flag_filter)
-            .transform(self.keep_dim_val_attr_columns)
+            .transform(self.keep_dim_val_attr_columns, additional_columns)
         )
 
-    def gen_gold_sws_data(self) -> DataFrame:
-        return self.read_bronze_data().transform(self.keep_dim_val_attr_columns)
+    def gen_gold_sws_data(self, additional_columns: List[str] = []) -> DataFrame:
+        return self.read_bronze_data().transform(
+            self.keep_dim_val_attr_columns, additional_columns
+        )
 
-    def gen_gold_sws_validated_data(self) -> DataFrame:
-        return self.read_silver_data().transform(self.keep_dim_val_attr_columns)
+    def gen_gold_sws_validated_data(
+        self, additional_columns: List[str] = []
+    ) -> DataFrame:
+        return self.read_silver_data().transform(
+            self.keep_dim_val_attr_columns, additional_columns
+        )
 
     def write_gold_sws_validated_data_to_iceberg_and_csv(
         self, df: DataFrame

{sws_spark_dissemination_helper-0.0.179.dist-info → sws_spark_dissemination_helper-0.0.181.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.179
+Version: 0.0.181
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://github.com/un-fao/fao-sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.179.dist-info → sws_spark_dissemination_helper-0.0.181.dist-info}/RECORD

@@ -1,13 +1,13 @@
 sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py,sha256=N0eQ2LXtpPeZQCWYi85sMLmpXRzLA2erECiba8tqOAY,29595
 sws_spark_dissemination_helper/SWSDatatablesExportHelper.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sws_spark_dissemination_helper/SWSEasyIcebergSparkHelper.py,sha256=csqKyYglBkJSBvEkEa1_keHarZZAIJHaV0d64gGJy98,26379
-sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py,sha256=IbJ9h1JKhuEMvfnkfUq2crrdzfa9BeuQualYVvFp1NA,27364
+sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py,sha256=atQFiY5Mmo-rzHY7WVWg-Guvg8i1ZcaaoKE4ymTaKdE,27750
 sws_spark_dissemination_helper/SWSPostgresSparkReader.py,sha256=V_rH4UYoFZfMUc82U-KxeL_o8F44HnMHfLLXoyNxHxs,20016
 sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py,sha256=EXpqPHbxld8MPShE6Vo8h4y1vpBt_BjMtS4RPJcPeTU,26355
 sws_spark_dissemination_helper/__init__.py,sha256=42TPbk7KxAud_qY3Sr_F4F7VjyofUlxEJkUXAFQsjRo,327
 sws_spark_dissemination_helper/constants.py,sha256=cVjTS3xbJNKz-1i7c1dJk2PcOZzQhvuHUp9i4PNIPh4,14055
 sws_spark_dissemination_helper/utils.py,sha256=Ge8zXsUIcvFihALDNLF5kCu_tAdRQUE04xE6Yn9xQF4,22008
-sws_spark_dissemination_helper-0.0.179.dist-info/METADATA,sha256=lma0Z7whiF6IMbknHWIsgiI8otJCgfnPoCaI0Wx9Odo,2822
-sws_spark_dissemination_helper-0.0.179.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-sws_spark_dissemination_helper-0.0.179.dist-info/licenses/LICENSE,sha256=zFzeb_j_6pXEHwH8Z0OpIkKFJk7vmhZjdem-K0d4zU4,1073
-sws_spark_dissemination_helper-0.0.179.dist-info/RECORD,,
+sws_spark_dissemination_helper-0.0.181.dist-info/METADATA,sha256=gszaR8nx1Aj_dWKKiUsQbCVpYIUZvIkq0BJ3DlfxYyk,2822
+sws_spark_dissemination_helper-0.0.181.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+sws_spark_dissemination_helper-0.0.181.dist-info/licenses/LICENSE,sha256=zFzeb_j_6pXEHwH8Z0OpIkKFJk7vmhZjdem-K0d4zU4,1073
+sws_spark_dissemination_helper-0.0.181.dist-info/RECORD,,