sws-spark-dissemination-helper 0.0.172__py3-none-any.whl → 0.0.176__py3-none-any.whl

sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py

@@ -87,7 +87,12 @@ class SWSGoldIcebergSparkHelper:
         return df.filter(col("diss_flag"))
 
     def keep_dim_val_attr_columns(self, df: DataFrame):
-        return df.select(*self.cols_to_keep_sws)
+        cols_to_keep_sws = self.cols_to_keep_sws
+        if "note" in df.columns:
+            cols_to_keep_sws = cols_to_keep_sws + ["note"]
+        if "unit_of_measure_symbol" in df.columns:
+            cols_to_keep_sws = cols_to_keep_sws + ["unit_of_measure_symbol"]
+        return df.select(*cols_to_keep_sws)
 
     def read_bronze_data(self) -> DataFrame:
         return self.spark.read.option("tag", self.tag_name).table(
@@ -667,3 +672,29 @@ class SWSGoldIcebergSparkHelper:
         logging.debug(f"Tag with Added csv Table: {tag}")
 
         return df
+
+
+1
+frozenset({"9", "1", "0", "4", "7", "3", "2", "6", "8", "5"})
+1
+1
+2
+frozenset({"9", "1", "0", "4", "7", "3", "2", "6", "8", "5"})
+2
+1
+1
+frozenset({"9", "1", "0", "4", "7", "3", "2", "6", "8", "5"})
+1
+1
+2
+frozenset({"9", "1", "0", "4", "7", "3", "2", "6", "8", "5"})
+2
+1
+1
+frozenset({"9", "1", "0", "4", "7", "3", "2", "6", "8", "5"})
+1
+1
+1
+frozenset({"9", "1", "0", "4", "7", "3", "2", "6", "8", "5"})
+1
+1

sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py

@@ -103,7 +103,7 @@ class SWSSilverIcebergSparkHelper:
         # The diss_flag column is needed to initialize the condition expression
         # The note column will contain the eventual reasons why diss_flag has been set to false
         return df.withColumn("diss_flag", lit(True)).withColumn(
-            "note", lit([]).cast(ArrayType(StringType()))
+            "diss_note", lit([]).cast(ArrayType(StringType()))
         )
 
     def read_bronze_data(self) -> DataFrame:
@@ -182,7 +182,7 @@ class SWSSilverIcebergSparkHelper:
                 for column in cols_to_select
                 if column.lower()
                 not in (
-                    "note",
+                    "diss_note",
                     f"{col_name_lower}_start_date",
                     f"{col_name_lower}_end_date",
                 )
@@ -203,36 +203,36 @@ class SWSSilverIcebergSparkHelper:
                 .withColumn("valid_new_start_year", col("sy.new_code").isNotNull())
                 .withColumn("valid_new_end_year", col("ey.new_code").isNotNull())
                 .withColumn(
-                    "new_note",
+                    "new_diss_note",
                     F.when(
                         col("valid_new_start_year"),
                         F.array_append(
-                            col("d.note"),
+                            col("d.diss_note"),
                             F.concat(
-                                col("sy.note"),
+                                col("sy.diss_note"),
                                 lit(" from "),
                                 col("sy.old_code"),
                                 lit(" to "),
                                 col("sy.new_code"),
                             ),
                         ),
-                    ).otherwise(col("d.note")),
+                    ).otherwise(col("d.diss_note")),
                 )
                 .withColumn(
-                    "new_note",
+                    "new_diss_note",
                     F.when(
                         col("valid_new_end_year"),
                         F.array_append(
-                            col("new_note"),
+                            col("new_diss_note"),
                             F.concat(
-                                col("ey.note"),
+                                col("ey.diss_note"),
                                 lit(" from "),
                                 col("ey.old_code"),
                                 lit(" to "),
                                 col("ey.new_code"),
                             ),
                         ),
-                    ).otherwise(col("new_note")),
+                    ).otherwise(col("new_diss_note")),
                 )
                 .withColumn(
                     f"new_{col_name}_start_date",
@@ -249,7 +249,7 @@ class SWSSilverIcebergSparkHelper:
                 )
                 .select(
                     *cols_to_select,
-                    col("new_note").alias("note"),
+                    col("new_diss_note").alias("diss_note"),
                     col(f"new_{col_name}_start_date").alias(f"{col_name}_start_date"),
                     col(f"new_{col_name}_end_date").alias(f"{col_name}_end_date"),
                 )
@@ -270,15 +270,15 @@ class SWSSilverIcebergSparkHelper:
                 start_date_condition & end_date_condition,
             )
             .withColumn("diss_flag", col("diss_flag") & col("condition_result"))
-            # In case the condition is satisfied update diss_flag accordingly and append a note indicating the reason for the observation exclusion from the dissemination
+            # In case the condition is satisfied update diss_flag accordingly and append a diss_note indicating the reason for the observation exclusion from the dissemination
             .withColumn(
-                "note",
+                "diss_note",
                 F.when(
                     ~col("condition_result"),
                     F.array_append(
-                        col("note"), lit(f"{col_type} out of time validity range")
+                        col("diss_note"), lit(f"{col_type} out of time validity range")
                     ),
-                ).otherwise(col("note")),
+                ).otherwise(col("diss_note")),
             )
             .drop("condition_result")
         )
@@ -390,7 +390,7 @@ class SWSSilverIcebergSparkHelper:
             col_name (str): The DataFrame column name on which to apply the filter
 
         Returns:
-            DataFrame: The DataFrame with updated `diss_flag` and `note` columns based on the check outcome
+            DataFrame: The DataFrame with updated `diss_flag` and `diss_note` columns based on the check outcome
         """
 
         # Remove the duplicates that may be in the tables
@@ -428,14 +428,14 @@ class SWSSilverIcebergSparkHelper:
                 col("diss_flag") & col("condition_result"),
             )
             .withColumn(
-                "note",
+                "diss_note",
                 F.when(
                     ~col("condition_result"),
                     F.array_append(
-                        col("note"),
+                        col("diss_note"),
                         lit(f"{col_type} not disseminated for this domain"),
                     ),
-                ).otherwise(col("note")),
+                ).otherwise(col("diss_note")),
             )
             .drop("condition_result")
         )
@@ -522,16 +522,16 @@ class SWSSilverIcebergSparkHelper:
                             col("diss_flag") & col("condition_result"),
                         )
                         .withColumn(
-                            "note",
+                            "diss_note",
                             F.when(
                                 ~col("condition_result"),
                                 F.array_append(
-                                    col("note"),
+                                    col("diss_note"),
                                     lit(
                                         f"not disseminated according to exception with note: {row_exception['note']}"
                                     ),
                                 ),
-                            ).otherwise(col("note")),
+                            ).otherwise(col("diss_note")),
                         )
                         .drop("condition_result")
                     )
@@ -616,7 +616,7 @@ class SWSSilverIcebergSparkHelper:
 
         df = (
             df.withColumn("metadata", F.to_json(col("metadata")))
-            .withColumn("note", F.to_json(col("note")))
+            .withColumn("diss_note", F.to_json(col("diss_note")))
             .coalesce(1)
         )
 
@@ -650,7 +650,7 @@ class SWSSilverIcebergSparkHelper:
                 "value",
                 *self.flag_columns,
                 "diss_flag",
-                "note",
+                "diss_note",
             ],
         )
         tag = upsert_disseminated_table(

sws_spark_dissemination_helper/utils.py

@@ -363,26 +363,26 @@ def map_codes_and_remove_null_duplicates(
             "diss_flag", F.when(col("delete"), lit(False)).otherwise(col("diss_flag"))
         )
         .withColumn(
-            "note",
+            "diss_note",
             F.when(
                 col("delete"),
                 F.array_append(
-                    col("note"),
+                    col("diss_note"),
                     lit(
                         f"The observation is not disseminated according to the Mapping - Code correction table"
                     ),
                 ),
-            ).otherwise(col("note")),
+            ).otherwise(col("diss_note")),
         )
         # Add mapping message to notes
         .withColumn(
-            "note",
+            "diss_note",
             F.when(
                 ~col("is_duplicate")
                 & col("new_dim_code").isNotNull()
                 & (col("new_dim_code") != lit("")),
                 F.array_append(
-                    col("note"),
+                    col("diss_note"),
                     F.concat(
                         lit(f"Dimension {col_name} code was changed from "),
                         col(col_name),
@@ -390,7 +390,7 @@ def map_codes_and_remove_null_duplicates(
                         col("new_dim_code"),
                     ),
                 ),
-            ).otherwise(col("note")),
+            ).otherwise(col("diss_note")),
         )
         .withColumn(
             col_name,
@@ -409,18 +409,18 @@ def map_codes_and_remove_null_duplicates(
             ).otherwise(col("diss_flag")),
         )
         .withColumn(
-            "note",
+            "diss_note",
             F.when(
                 col("is_duplicate")
                 & col("new_dim_code").isNotNull()
                 & (col("new_dim_code") != lit("")),
                 F.array_append(
-                    col("note"),
+                    col("diss_note"),
                     lit(
                         f"The code correction was not applied to avoid observation duplications"
                     ),
                 ),
-            ).otherwise(col("note")),
+            ).otherwise(col("diss_note")),
         )
         # Check the domain specific multiplier first and then the standard multiplier
         .withColumn("value", col("value") * F.coalesce(col("multiplier"), lit(1)))

{sws_spark_dissemination_helper-0.0.172.dist-info → sws_spark_dissemination_helper-0.0.176.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.172
+Version: 0.0.176
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://github.com/un-fao/fao-sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.172.dist-info → sws_spark_dissemination_helper-0.0.176.dist-info}/RECORD

@@ -1,13 +1,13 @@
 sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py,sha256=N0eQ2LXtpPeZQCWYi85sMLmpXRzLA2erECiba8tqOAY,29595
 sws_spark_dissemination_helper/SWSDatatablesExportHelper.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sws_spark_dissemination_helper/SWSEasyIcebergSparkHelper.py,sha256=csqKyYglBkJSBvEkEa1_keHarZZAIJHaV0d64gGJy98,26379
-sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py,sha256=0dxbVkrhdaASapEffF5PFcgKwAMyJoWBxzgymjZ4JyY,25049
+sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py,sha256=wJgtpD0DqaGNWu_V5lcPIaPRBFimV395gn8Fk7Ul9bY,25723
 sws_spark_dissemination_helper/SWSPostgresSparkReader.py,sha256=KpG8gp8Ai9pHDiKhUOTcXWxxmFGeKEE3XKlI_Y-SveU,18453
-sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py,sha256=qioLv3SlJEfk0LzTiwfXRtZXVImPOJUeh9k1XwHC-pA,26225
+sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py,sha256=EXpqPHbxld8MPShE6Vo8h4y1vpBt_BjMtS4RPJcPeTU,26355
 sws_spark_dissemination_helper/__init__.py,sha256=42TPbk7KxAud_qY3Sr_F4F7VjyofUlxEJkUXAFQsjRo,327
 sws_spark_dissemination_helper/constants.py,sha256=vQmalAqInwPAybgJOfYx99jn47KsKp8jeD8eqmjw-Rs,13471
-sws_spark_dissemination_helper/utils.py,sha256=G7lQqNRrvqZpgm9WmddD7fWsI8IVn09x1p3cV3458EA,21963
-sws_spark_dissemination_helper-0.0.172.dist-info/METADATA,sha256=ffSpRvz94JsDgOTUB6YVXFMukTSdu_Ve70d_zE0a3YA,2822
-sws_spark_dissemination_helper-0.0.172.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-sws_spark_dissemination_helper-0.0.172.dist-info/licenses/LICENSE,sha256=zFzeb_j_6pXEHwH8Z0OpIkKFJk7vmhZjdem-K0d4zU4,1073
-sws_spark_dissemination_helper-0.0.172.dist-info/RECORD,,
+sws_spark_dissemination_helper/utils.py,sha256=Ge8zXsUIcvFihALDNLF5kCu_tAdRQUE04xE6Yn9xQF4,22008
+sws_spark_dissemination_helper-0.0.176.dist-info/METADATA,sha256=3mtkAYmFx0fjK60OEEAbhBSOH4605meXWJX1cjmvGHo,2822
+sws_spark_dissemination_helper-0.0.176.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+sws_spark_dissemination_helper-0.0.176.dist-info/licenses/LICENSE,sha256=zFzeb_j_6pXEHwH8Z0OpIkKFJk7vmhZjdem-K0d4zU4,1073
+sws_spark_dissemination_helper-0.0.176.dist-info/RECORD,,