sws-spark-dissemination-helper 0.0.179__py3-none-any.whl → 0.0.187__py3-none-any.whl

sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py

@@ -88,10 +88,13 @@ class SWSGoldIcebergSparkHelper:
     def apply_diss_flag_filter(self, df: DataFrame) -> DataFrame:
         return df.filter(col("diss_flag"))
 
-    def keep_dim_val_attr_columns(self, df: DataFrame):
+    def keep_dim_val_attr_columns(
+        self, df: DataFrame, additional_columns: List[str] = []
+    ):
         cols_to_keep_sws = self.cols_to_keep_sws
-        if "note" in df.columns:
-            cols_to_keep_sws = cols_to_keep_sws + ["note"]
+        for additional_column in additional_columns:
+            if additional_column in df.columns:
+                cols_to_keep_sws = cols_to_keep_sws + [additional_column]
         if "unit_of_measure_symbol" in df.columns:
             cols_to_keep_sws = cols_to_keep_sws + ["unit_of_measure_symbol"]
         return df.select(*cols_to_keep_sws)
@@ -137,7 +140,7 @@ class SWSGoldIcebergSparkHelper:
             F.round(
                 F.col("value").cast("FLOAT") * F.pow(10, F.col("display_decimals")), 0
             )
-            / F.pow(10, F.col("dec")).cast("STRING"),
+            / F.pow(10, F.col("display_decimals")).cast("STRING"),
         )
 
         # F.round(
@@ -156,18 +159,26 @@ class SWSGoldIcebergSparkHelper:
             self.iceberg_tables.SILVER.iceberg_id
         )
 
-    def gen_gold_sws_disseminated_data(self) -> DataFrame:
+    def gen_gold_sws_disseminated_data(
+        self, additional_columns: List[str] = []
+    ) -> DataFrame:
         return (
             self.read_silver_data()
             .transform(self.apply_diss_flag_filter)
-            .transform(self.keep_dim_val_attr_columns)
+            .transform(self.keep_dim_val_attr_columns, additional_columns)
         )
 
-    def gen_gold_sws_data(self) -> DataFrame:
-        return self.read_bronze_data().transform(self.keep_dim_val_attr_columns)
+    def gen_gold_sws_data(self, additional_columns: List[str] = []) -> DataFrame:
+        return self.read_bronze_data().transform(
+            self.keep_dim_val_attr_columns, additional_columns
+        )
 
-    def gen_gold_sws_validated_data(self) -> DataFrame:
-        return self.read_silver_data().transform(self.keep_dim_val_attr_columns)
+    def gen_gold_sws_validated_data(
+        self, additional_columns: List[str] = []
+    ) -> DataFrame:
+        return self.read_silver_data().transform(
+            self.keep_dim_val_attr_columns, additional_columns
+        )
 
     def write_gold_sws_validated_data_to_iceberg_and_csv(
         self, df: DataFrame

sws_spark_dissemination_helper/SWSPostgresSparkReader.py

@@ -468,7 +468,7 @@ class SWSPostgresSparkReader:
                 correct_domain_filter, domain=domain_code, unique_columns=["code"]
             )
             for col_type in mapping_dim_col_name_type.values()
-            if col_type != "other"
+            if col_type not in ("year", "other")
         }
 
     def import_diss_exceptions_datatable(

sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py

@@ -209,7 +209,7 @@ class SWSSilverIcebergSparkHelper:
                         F.array_append(
                             col("d.diss_note"),
                             F.concat(
-                                col("sy.diss_note"),
+                                col("sy.note"),
                                 lit(" from "),
                                 col("sy.old_code"),
                                 lit(" to "),
@@ -225,7 +225,7 @@ class SWSSilverIcebergSparkHelper:
                         F.array_append(
                             col("new_diss_note"),
                             F.concat(
-                                col("ey.diss_note"),
+                                col("ey.note"),
                                 lit(" from "),
                                 col("ey.old_code"),
                                 lit(" to "),

sws_spark_dissemination_helper/constants.py

@@ -168,7 +168,7 @@ class DatasetTables:
         self.OBSERVATION = self.__SWSTable(
             postgres_id=f"{self.__dataset_id}.observation",
             iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.{self.__dataset_id}_observation",
-            schema="id BIGINT, observation_coordinates BIGINT, version INT, value FLOAT, flag_obs_status STRING, flag_method STRING, created_on TIMESTAMP, created_by INT, replaced_on TIMESTAMP",
+            schema="id BIGINT, observation_coordinates BIGINT, version INT, value STRING, flag_obs_status STRING, flag_method STRING, created_on TIMESTAMP, created_by INT, replaced_on TIMESTAMP",
         )
         self.OBSERVATION_COORDINATE = self.__SWSTable(
             postgres_id=f"{self.__dataset_id}.observation_coordinate",

{sws_spark_dissemination_helper-0.0.179.dist-info → sws_spark_dissemination_helper-0.0.187.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.179
+Version: 0.0.187
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://github.com/un-fao/fao-sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>
@@ -49,7 +49,7 @@ Requires-Dist: pytz==2025.2
 Requires-Dist: requests==2.32.3
 Requires-Dist: s3transfer>=0.11.2
 Requires-Dist: six==1.17.0
-Requires-Dist: sws-api-client==2.3.0
+Requires-Dist: sws-api-client==2.7.3
 Requires-Dist: typing-extensions>=4.12.2
 Requires-Dist: tzdata==2025.2
 Requires-Dist: urllib3==1.26.20

{sws_spark_dissemination_helper-0.0.179.dist-info → sws_spark_dissemination_helper-0.0.187.dist-info}/RECORD

@@ -1,13 +1,13 @@
 sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py,sha256=N0eQ2LXtpPeZQCWYi85sMLmpXRzLA2erECiba8tqOAY,29595
 sws_spark_dissemination_helper/SWSDatatablesExportHelper.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sws_spark_dissemination_helper/SWSEasyIcebergSparkHelper.py,sha256=csqKyYglBkJSBvEkEa1_keHarZZAIJHaV0d64gGJy98,26379
-sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py,sha256=IbJ9h1JKhuEMvfnkfUq2crrdzfa9BeuQualYVvFp1NA,27364
-sws_spark_dissemination_helper/SWSPostgresSparkReader.py,sha256=V_rH4UYoFZfMUc82U-KxeL_o8F44HnMHfLLXoyNxHxs,20016
-sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py,sha256=EXpqPHbxld8MPShE6Vo8h4y1vpBt_BjMtS4RPJcPeTU,26355
+sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py,sha256=atQFiY5Mmo-rzHY7WVWg-Guvg8i1ZcaaoKE4ymTaKdE,27750
+sws_spark_dissemination_helper/SWSPostgresSparkReader.py,sha256=qoO___xL1g1iH_KkJ0opLvtNJGU2Dm6bUn-jWem5v2U,20030
+sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py,sha256=3l5zkEWksnEC-R4mJi8JEHL3ylCMbkMD9a0qbdZQU5E,26345
 sws_spark_dissemination_helper/__init__.py,sha256=42TPbk7KxAud_qY3Sr_F4F7VjyofUlxEJkUXAFQsjRo,327
-sws_spark_dissemination_helper/constants.py,sha256=cVjTS3xbJNKz-1i7c1dJk2PcOZzQhvuHUp9i4PNIPh4,14055
+sws_spark_dissemination_helper/constants.py,sha256=MzuC7pqsXF89r-FK7hhmWaZSk5x3GB_YPVSfuK3NYVY,14056
 sws_spark_dissemination_helper/utils.py,sha256=Ge8zXsUIcvFihALDNLF5kCu_tAdRQUE04xE6Yn9xQF4,22008
-sws_spark_dissemination_helper-0.0.179.dist-info/METADATA,sha256=lma0Z7whiF6IMbknHWIsgiI8otJCgfnPoCaI0Wx9Odo,2822
-sws_spark_dissemination_helper-0.0.179.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-sws_spark_dissemination_helper-0.0.179.dist-info/licenses/LICENSE,sha256=zFzeb_j_6pXEHwH8Z0OpIkKFJk7vmhZjdem-K0d4zU4,1073
-sws_spark_dissemination_helper-0.0.179.dist-info/RECORD,,
+sws_spark_dissemination_helper-0.0.187.dist-info/METADATA,sha256=PPrDi-8X1HkcAjYs92VJRaAvcf27I3Aw0wljIs1UMO8,2822
+sws_spark_dissemination_helper-0.0.187.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+sws_spark_dissemination_helper-0.0.187.dist-info/licenses/LICENSE,sha256=zFzeb_j_6pXEHwH8Z0OpIkKFJk7vmhZjdem-K0d4zU4,1073
+sws_spark_dissemination_helper-0.0.187.dist-info/RECORD,,