sws-spark-dissemination-helper 0.0.166__tar.gz → 0.0.176__tar.gz

{sws_spark_dissemination_helper-0.0.166 → sws_spark_dissemination_helper-0.0.176}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.166
+Version: 0.0.176
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://github.com/un-fao/fao-sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>
@@ -31,27 +31,27 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.9
 Requires-Dist: annotated-types==0.7.0
-Requires-Dist: boto3>=1.36.18
-Requires-Dist: botocore>=1.36.18
+Requires-Dist: boto3>=1.40.0
+Requires-Dist: botocore>=1.40.0
 Requires-Dist: certifi==2025.1.31
 Requires-Dist: charset-normalizer==3.4.1
-Requires-Dist: idna==3.10
+Requires-Dist: idna>=3.10
 Requires-Dist: jmespath==1.0.1
 Requires-Dist: numpy==2.0.2
-Requires-Dist: pandas==2.2.3
+Requires-Dist: pandas==2.3.3
 Requires-Dist: py4j==0.10.9.7
 Requires-Dist: pydantic-core==2.27.2
 Requires-Dist: pydantic==2.10.6
 Requires-Dist: pyspark==3.5.4
 Requires-Dist: python-dateutil==2.9.0.post0
 Requires-Dist: python-dotenv==0.19.2
-Requires-Dist: pytz==2025.1
+Requires-Dist: pytz==2025.2
 Requires-Dist: requests==2.32.3
-Requires-Dist: s3transfer==0.11.2
+Requires-Dist: s3transfer>=0.11.2
 Requires-Dist: six==1.17.0
-Requires-Dist: sws-api-client==1.5.3
-Requires-Dist: typing-extensions==4.12.2
-Requires-Dist: tzdata==2025.1
+Requires-Dist: sws-api-client==2.3.0
+Requires-Dist: typing-extensions>=4.12.2
+Requires-Dist: tzdata==2025.2
 Requires-Dist: urllib3==1.26.20
 Description-Content-Type: text/markdown
 

{sws_spark_dissemination_helper-0.0.166 → sws_spark_dissemination_helper-0.0.176}/pyproject.toml

@@ -4,30 +4,30 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.166"
+version = "0.0.176"
 dependencies = [
     "annotated-types==0.7.0",
-    "boto3>=1.36.18",
-    "botocore>=1.36.18",
+    "boto3>=1.40.0",
+    "botocore>=1.40.0",
     "certifi==2025.1.31",
     "charset-normalizer==3.4.1",
-    "idna==3.10",
+    "idna>=3.10",
     "jmespath==1.0.1",
     "numpy==2.0.2",
-    "pandas==2.2.3",
+    "pandas==2.3.3",
     "py4j==0.10.9.7",
     "pydantic==2.10.6",
     "pydantic_core==2.27.2",
     "pyspark==3.5.4",
     "python-dateutil==2.9.0.post0",
     "python-dotenv==0.19.2",
-    "pytz==2025.1",
+    "pytz==2025.2",
     "requests==2.32.3",
-    "s3transfer==0.11.2",
+    "s3transfer>=0.11.2",
     "six==1.17.0",
-    "sws_api_client==1.5.3",
-    "typing_extensions==4.12.2",
-    "tzdata==2025.1",
+    "sws_api_client==2.3.0",
+    "typing_extensions>=4.12.2",
+    "tzdata==2025.2",
     "urllib3==1.26.20"
 ]
 requires-python = ">=3.9"

{sws_spark_dissemination_helper-0.0.166 → sws_spark_dissemination_helper-0.0.176}/src/sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py

@@ -158,7 +158,7 @@ class SWSBronzeIcebergSparkHelper:
 
         return dfs_dimension
 
-    def _prepare_element_uom(self) -> DataFrame:
+    def _prepare_element_uom(self) -> Union[DataFrame, None]:
         """Prepare the element and unit of measure join."""
 
         # Get the element DataFrame
@@ -170,23 +170,24 @@ class SWSBronzeIcebergSparkHelper:
             if dimension_column == self.element_column
         )
 
-        # Join the element and the unit_of_measure
-        df_element_uom = (
-            df_element.alias("e")
-            .join(
-                self.df_unit_of_measure.alias("u"),
-                col("e.unit_of_measure") == col("u.id"),
-            )
-            .select(
-                col("e.code").alias("element_code"),
-                col("u.code").alias("unit_of_measure"),
-                col("u.symbol").alias("unit_of_measure_symbol"),
-                col("u.base_unit").alias("unit_of_measure_base_unit"),
-                col("u.multiplier").alias("unit_of_measure_multiplier"),
+        if any("unit_of_measure" == column.lower() for column in df_element.columns):
+            # Join the element and the unit_of_measure
+            df_element_uom = (
+                df_element.alias("e")
+                .join(
+                    self.df_unit_of_measure.alias("u"),
+                    col("e.unit_of_measure") == col("u.id"),
+                )
+                .select(
+                    col("e.code").alias("element_code"),
+                    col("u.code").alias("unit_of_measure"),
+                    col("u.symbol").alias("unit_of_measure_symbol"),
+                    col("u.base_unit").alias("unit_of_measure_base_unit"),
+                    col("u.multiplier").alias("unit_of_measure_multiplier"),
+                )
             )
-        )
 
-        return df_element_uom
+            return df_element_uom
 
     def _gen_denormalized_observation(self) -> DataFrame:
         """Original query upon which the below computation is based
@@ -278,15 +279,16 @@ class SWSBronzeIcebergSparkHelper:
                 .withColumnRenamed("code", dimension_column)
             )
 
-        df_intermediate = (
-            df_intermediate.alias("d")
-            .join(
-                F.broadcast(df_element_uom).alias("e"),
-                col(f"d.{self.element_column}") == col("e.element_code"),
-                "left",
+        if df_element_uom is not None:
+            df_intermediate = (
+                df_intermediate.alias("d")
+                .join(
+                    F.broadcast(df_element_uom).alias("e"),
+                    col(f"d.{self.element_column}") == col("e.element_code"),
+                    "left",
+                )
+                .drop("element_code")
             )
-            .drop("element_code")
-        )
 
         df_obs_denorm = df_intermediate
 
@@ -364,16 +366,17 @@ class SWSBronzeIcebergSparkHelper:
             )
             logging.debug(f"After join count: {df_obs_denorm.count()}")
 
-        df_obs_denorm = (
-            df_obs_denorm.alias("d")
-            .join(
-                F.broadcast(df_element_uom).alias("e"),
-                col(f"d.{self.element_column}") == col("e.element_code"),
-                "left",
+        if df_element_uom is not None:
+            df_obs_denorm = (
+                df_obs_denorm.alias("d")
+                .join(
+                    F.broadcast(df_element_uom).alias("e"),
+                    col(f"d.{self.element_column}") == col("e.element_code"),
+                    "left",
+                )
+                .drop("element_code")
             )
-            .drop("element_code")
-        )
-        logging.debug(f"After uom count: {df_obs_denorm.count()}")
+            logging.debug(f"After uom count: {df_obs_denorm.count()}")
 
         return df_obs_denorm
 
@@ -766,3 +769,29 @@ class SWSBronzeIcebergSparkHelper:
         logging.debug(f"Tag with Added csv Table: {tag}")
 
         logging.info("Bronze Disseminated tag with selection successfully written")
+
+
+1
+frozenset({"8", "4", "2", "5", "9", "1", "7", "6", "0", "3"})
+1
+1
+2
+frozenset({"8", "4", "2", "5", "9", "1", "7", "6", "0", "3"})
+2
+1
+1
+frozenset({"8", "4", "2", "5", "9", "1", "7", "6", "0", "3"})
+1
+1
+2
+frozenset({"8", "4", "2", "5", "9", "1", "7", "6", "0", "3"})
+2
+1
+1
+frozenset({"8", "4", "2", "5", "9", "1", "7", "6", "0", "3"})
+1
+1
+1
+frozenset({"8", "4", "2", "5", "9", "1", "7", "6", "0", "3"})
+1
+1

{sws_spark_dissemination_helper-0.0.166 → sws_spark_dissemination_helper-0.0.176}/src/sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py

@@ -87,7 +87,12 @@ class SWSGoldIcebergSparkHelper:
         return df.filter(col("diss_flag"))
 
     def keep_dim_val_attr_columns(self, df: DataFrame):
-        return df.select(*self.cols_to_keep_sws)
+        cols_to_keep_sws = self.cols_to_keep_sws
+        if "note" in df.columns:
+            cols_to_keep_sws = cols_to_keep_sws + ["note"]
+        if "unit_of_measure_symbol" in df.columns:
+            cols_to_keep_sws = cols_to_keep_sws + ["unit_of_measure_symbol"]
+        return df.select(*cols_to_keep_sws)
 
     def read_bronze_data(self) -> DataFrame:
         return self.spark.read.option("tag", self.tag_name).table(
@@ -296,6 +301,35 @@ class SWSGoldIcebergSparkHelper:
 
         return df
 
+    def write_gold_faostat_unfiltered_data_to_iceberg_and_csv(
+        self, df: DataFrame
+    ) -> DataFrame:
+        """The expected input to this function is the output of the sws disseminated function"""
+        df.writeTo(
+            self.iceberg_tables.GOLD_FAOSTAT_UNFILTERED.iceberg_id
+        ).createOrReplace()
+
+        logging.info(
+            f"Gold FAOSTAT unfiltered table written to {self.iceberg_tables.GOLD_FAOSTAT.iceberg_id}"
+        )
+
+        self.spark.sql(
+            f"ALTER TABLE {self.iceberg_tables.GOLD_FAOSTAT_UNFILTERED.iceberg_id} CREATE OR REPLACE TAG `{self.tag_name}`"
+        )
+
+        logging.info(f"gold FAOSTAT unfiltered tag '{self.tag_name}' created")
+
+        df_1 = df.coalesce(1)
+
+        save_cache_csv(
+            df=df_1,
+            bucket=self.bucket,
+            prefix=self.iceberg_tables.GOLD_FAOSTAT_UNFILTERED.csv_prefix,
+            tag_name=self.tag_name,
+        )
+
+        return df
+
     def write_gold_sws_validated_sws_dissemination_tag(
         self, df: DataFrame, tags: Tags
     ) -> DataFrame:
@@ -589,3 +623,78 @@ class SWSGoldIcebergSparkHelper:
         logging.debug(f"Tag with Added csv Table: {tag}")
 
         return df
+
+    def write_gold_faostat_unfiltered_dissemination_tag(
+        self, df: DataFrame, tags: Tags
+    ) -> DataFrame:
+        # Get or create a new tag
+        tag = get_or_create_tag(tags, self.dataset_id, self.tag_name, self.tag_name)
+        logging.debug(f"Tag: {tag}")
+
+        new_iceberg_table = BaseDisseminatedTagTable(
+            id=f"{self.domain_code.lower()}_gold_faostat_unfiltered_iceberg",
+            name=f"{self.domain_code} gold FAOSTAT unfiltered Iceberg",
+            description="Gold table containing all the tag data in FAOSTAT format",
+            layer=TableLayer.GOLD,
+            private=True,
+            type=TableType.ICEBERG,
+            database=IcebergDatabases.GOLD_DATABASE,
+            table=self.iceberg_tables.GOLD_FAOSTAT_UNFILTERED.table,
+            path=self.iceberg_tables.GOLD_FAOSTAT_UNFILTERED.path,
+            structure={"columns": df.schema.jsonValue()["fields"]},
+        )
+        tag = upsert_disseminated_table(
+            sws_tags=tags,
+            tag=tag,
+            dataset_id=self.dataset_id,
+            tag_name=self.tag_name,
+            table=new_iceberg_table,
+        )
+        logging.debug(f"Tag with Added Iceberg Table: {tag}")
+
+        new_diss_table = BaseDisseminatedTagTable(
+            id=f"{self.domain_code.lower()}_gold_faostat_unfiltered_csv",
+            name=f"{self.domain_code} gold FAOSTAT unfiltered csv",
+            description="Gold table containing the tag data in FAOSTAT format in csv",
+            layer=TableLayer.GOLD,
+            private=True,
+            type=TableType.CSV,
+            path=self.iceberg_tables.GOLD_FAOSTAT_UNFILTERED.csv_path,
+            structure={"columns": df.schema.jsonValue()["fields"]},
+        )
+        tag = upsert_disseminated_table(
+            sws_tags=tags,
+            tag=tag,
+            dataset_id=self.dataset_id,
+            tag_name=self.tag_name,
+            table=new_diss_table,
+        )
+        logging.debug(f"Tag with Added csv Table: {tag}")
+
+        return df
+
+
+1
+frozenset({"9", "1", "0", "4", "7", "3", "2", "6", "8", "5"})
+1
+1
+2
+frozenset({"9", "1", "0", "4", "7", "3", "2", "6", "8", "5"})
+2
+1
+1
+frozenset({"9", "1", "0", "4", "7", "3", "2", "6", "8", "5"})
+1
+1
+2
+frozenset({"9", "1", "0", "4", "7", "3", "2", "6", "8", "5"})
+2
+1
+1
+frozenset({"9", "1", "0", "4", "7", "3", "2", "6", "8", "5"})
+1
+1
+1
+frozenset({"9", "1", "0", "4", "7", "3", "2", "6", "8", "5"})
+1
+1

{sws_spark_dissemination_helper-0.0.166 → sws_spark_dissemination_helper-0.0.176}/src/sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py

@@ -103,7 +103,7 @@ class SWSSilverIcebergSparkHelper:
         # The diss_flag column is needed to initialize the condition expression
         # The note column will contain the eventual reasons why diss_flag has been set to false
         return df.withColumn("diss_flag", lit(True)).withColumn(
-            "note", lit([]).cast(ArrayType(StringType()))
+            "diss_note", lit([]).cast(ArrayType(StringType()))
         )
 
     def read_bronze_data(self) -> DataFrame:
@@ -182,7 +182,7 @@ class SWSSilverIcebergSparkHelper:
                 for column in cols_to_select
                 if column.lower()
                 not in (
-                    "note",
+                    "diss_note",
                     f"{col_name_lower}_start_date",
                     f"{col_name_lower}_end_date",
                 )
@@ -203,36 +203,36 @@ class SWSSilverIcebergSparkHelper:
                 .withColumn("valid_new_start_year", col("sy.new_code").isNotNull())
                 .withColumn("valid_new_end_year", col("ey.new_code").isNotNull())
                 .withColumn(
-                    "new_note",
+                    "new_diss_note",
                     F.when(
                         col("valid_new_start_year"),
                         F.array_append(
-                            col("d.note"),
+                            col("d.diss_note"),
                             F.concat(
-                                col("sy.note"),
+                                col("sy.diss_note"),
                                 lit(" from "),
                                 col("sy.old_code"),
                                 lit(" to "),
                                 col("sy.new_code"),
                             ),
                         ),
-                    ).otherwise(col("d.note")),
+                    ).otherwise(col("d.diss_note")),
                 )
                 .withColumn(
-                    "new_note",
+                    "new_diss_note",
                     F.when(
                         col("valid_new_end_year"),
                         F.array_append(
-                            col("new_note"),
+                            col("new_diss_note"),
                             F.concat(
-                                col("ey.note"),
+                                col("ey.diss_note"),
                                 lit(" from "),
                                 col("ey.old_code"),
                                 lit(" to "),
                                 col("ey.new_code"),
                             ),
                         ),
-                    ).otherwise(col("new_note")),
+                    ).otherwise(col("new_diss_note")),
                 )
                 .withColumn(
                     f"new_{col_name}_start_date",
@@ -249,7 +249,7 @@ class SWSSilverIcebergSparkHelper:
                 )
                 .select(
                     *cols_to_select,
-                    col("new_note").alias("note"),
+                    col("new_diss_note").alias("diss_note"),
                     col(f"new_{col_name}_start_date").alias(f"{col_name}_start_date"),
                     col(f"new_{col_name}_end_date").alias(f"{col_name}_end_date"),
                 )
@@ -270,15 +270,15 @@ class SWSSilverIcebergSparkHelper:
                 start_date_condition & end_date_condition,
             )
             .withColumn("diss_flag", col("diss_flag") & col("condition_result"))
-            # In case the condition is satisfied update diss_flag accordingly and append a note indicating the reason for the observation exclusion from the dissemination
+            # In case the condition is satisfied update diss_flag accordingly and append a diss_note indicating the reason for the observation exclusion from the dissemination
             .withColumn(
-                "note",
+                "diss_note",
                 F.when(
                     ~col("condition_result"),
                     F.array_append(
-                        col("note"), lit(f"{col_type} out of time validity range")
+                        col("diss_note"), lit(f"{col_type} out of time validity range")
                     ),
-                ).otherwise(col("note")),
+                ).otherwise(col("diss_note")),
             )
             .drop("condition_result")
         )
@@ -390,7 +390,7 @@ class SWSSilverIcebergSparkHelper:
             col_name (str): The DataFrame column name on which to apply the filter
 
         Returns:
-            DataFrame: The DataFrame with updated `diss_flag` and `note` columns based on the check outcome
+            DataFrame: The DataFrame with updated `diss_flag` and `diss_note` columns based on the check outcome
         """
 
         # Remove the duplicates that may be in the tables
@@ -428,14 +428,14 @@ class SWSSilverIcebergSparkHelper:
                 col("diss_flag") & col("condition_result"),
             )
             .withColumn(
-                "note",
+                "diss_note",
                 F.when(
                     ~col("condition_result"),
                     F.array_append(
-                        col("note"),
+                        col("diss_note"),
                         lit(f"{col_type} not disseminated for this domain"),
                     ),
-                ).otherwise(col("note")),
+                ).otherwise(col("diss_note")),
             )
             .drop("condition_result")
         )
@@ -522,16 +522,16 @@ class SWSSilverIcebergSparkHelper:
                             col("diss_flag") & col("condition_result"),
                         )
                         .withColumn(
-                            "note",
+                            "diss_note",
                             F.when(
                                 ~col("condition_result"),
                                 F.array_append(
-                                    col("note"),
+                                    col("diss_note"),
                                     lit(
                                         f"not disseminated according to exception with note: {row_exception['note']}"
                                     ),
                                 ),
-                            ).otherwise(col("note")),
+                            ).otherwise(col("diss_note")),
                         )
                         .drop("condition_result")
                     )
@@ -616,7 +616,7 @@ class SWSSilverIcebergSparkHelper:
 
         df = (
             df.withColumn("metadata", F.to_json(col("metadata")))
-            .withColumn("note", F.to_json(col("note")))
+            .withColumn("diss_note", F.to_json(col("diss_note")))
             .coalesce(1)
         )
 
@@ -650,7 +650,7 @@ class SWSSilverIcebergSparkHelper:
                 "value",
                 *self.flag_columns,
                 "diss_flag",
-                "note",
+                "diss_note",
             ],
         )
         tag = upsert_disseminated_table(

{sws_spark_dissemination_helper-0.0.166 → sws_spark_dissemination_helper-0.0.176}/src/sws_spark_dissemination_helper/constants.py

@@ -254,37 +254,37 @@ class IcebergTables:
         self.__tag_name = tag_name
 
         # TODO Fix later with a more appropriate DATABASE
-        self.DENORMALIZED_OBSERVATION = self._create_iceberg_table("BRONZE", suffix="denormalized_observation")
-        self.DENORMALIZED_METADATA = self._create_iceberg_table("BRONZE", suffix="denormalized_metadata")
-        self.GROUPED_METADATA = self._create_iceberg_table("BRONZE", suffix="grouped_metadata")
-        self.TABLE = self._create_iceberg_table("BRONZE")
-        self.TABLE_FILTERED = self._create_iceberg_table("BRONZE", suffix="filtered")
-        self.BRONZE = self._create_iceberg_table("BRONZE")
-        self.BRONZE_DISS_TAG = self._create_iceberg_table("BRONZE", suffix="diss_tag")
-        self.SILVER = self._create_iceberg_table("SILVER", prefix=domain)
+        self.DENORMALIZED_OBSERVATION = self.create_iceberg_table("BRONZE", suffix="denormalized_observation")
+        self.DENORMALIZED_METADATA = self.create_iceberg_table("BRONZE", suffix="denormalized_metadata")
+        self.GROUPED_METADATA = self.create_iceberg_table("BRONZE", suffix="grouped_metadata")
+        self.TABLE = self.create_iceberg_table("BRONZE")
+        self.TABLE_FILTERED = self.create_iceberg_table("BRONZE", suffix="filtered")
+        self.BRONZE = self.create_iceberg_table("BRONZE")
+        self.BRONZE_DISS_TAG = self.create_iceberg_table("BRONZE", suffix="diss_tag")
+        self.SILVER = self.create_iceberg_table("SILVER", prefix=domain)
 
         # GOLD tables with specific suffixes
-        self.GOLD_SWS = self._create_iceberg_table("GOLD", prefix=domain, suffix="sws")
-        self.GOLD_SDMX = self._create_iceberg_table(
+        self.GOLD_SWS = self.create_iceberg_table("GOLD", prefix=domain, suffix="sws")
+        self.GOLD_SDMX = self.create_iceberg_table(
             "GOLD", prefix=domain, suffix="sdmx_disseminated"
         )
-        self.GOLD_SWS_VALIDATED = self._create_iceberg_table(
+        self.GOLD_SWS_VALIDATED = self.create_iceberg_table(
             "GOLD", prefix=domain, suffix="sws_validated"
         )
-        self.GOLD_SWS_DISSEMINATED = self._create_iceberg_table(
+        self.GOLD_SWS_DISSEMINATED = self.create_iceberg_table(
             "GOLD", prefix=domain, suffix="sws_disseminated"
         )
-        self.GOLD_PRE_SDMX = self._create_iceberg_table(
+        self.GOLD_PRE_SDMX = self.create_iceberg_table(
             "GOLD", prefix=domain, suffix="pre_sdmx"
         )
-        self.GOLD_FAOSTAT = self._create_iceberg_table(
+        self.GOLD_FAOSTAT = self.create_iceberg_table(
             "GOLD", prefix=domain, suffix="faostat"
         )
-        self.GOLD_FAOSTAT_UNFILTERED = self._create_iceberg_table(
+        self.GOLD_FAOSTAT_UNFILTERED = self.create_iceberg_table(
             "GOLD", prefix=domain, suffix="faostat_unfiltered"
         )
 
-    def _create_iceberg_table(
+    def create_iceberg_table(
         self, level: str, prefix: str = "", suffix: str = ""
     ) -> IcebergTable:
         database = getattr(IcebergDatabases, f"{level}_DATABASE")

{sws_spark_dissemination_helper-0.0.166 → sws_spark_dissemination_helper-0.0.176}/src/sws_spark_dissemination_helper/utils.py

@@ -363,26 +363,26 @@ def map_codes_and_remove_null_duplicates(
             "diss_flag", F.when(col("delete"), lit(False)).otherwise(col("diss_flag"))
         )
         .withColumn(
-            "note",
+            "diss_note",
             F.when(
                 col("delete"),
                 F.array_append(
-                    col("note"),
+                    col("diss_note"),
                     lit(
                         f"The observation is not disseminated according to the Mapping - Code correction table"
                     ),
                 ),
-            ).otherwise(col("note")),
+            ).otherwise(col("diss_note")),
         )
         # Add mapping message to notes
         .withColumn(
-            "note",
+            "diss_note",
             F.when(
                 ~col("is_duplicate")
                 & col("new_dim_code").isNotNull()
                 & (col("new_dim_code") != lit("")),
                 F.array_append(
-                    col("note"),
+                    col("diss_note"),
                     F.concat(
                         lit(f"Dimension {col_name} code was changed from "),
                         col(col_name),
@@ -390,7 +390,7 @@ def map_codes_and_remove_null_duplicates(
                         col("new_dim_code"),
                     ),
                 ),
-            ).otherwise(col("note")),
+            ).otherwise(col("diss_note")),
         )
         .withColumn(
             col_name,
@@ -409,18 +409,18 @@ def map_codes_and_remove_null_duplicates(
             ).otherwise(col("diss_flag")),
         )
         .withColumn(
-            "note",
+            "diss_note",
             F.when(
                 col("is_duplicate")
                 & col("new_dim_code").isNotNull()
                 & (col("new_dim_code") != lit("")),
                 F.array_append(
-                    col("note"),
+                    col("diss_note"),
                     lit(
                         f"The code correction was not applied to avoid observation duplications"
                     ),
                 ),
-            ).otherwise(col("note")),
+            ).otherwise(col("diss_note")),
         )
         # Check the domain specific multiplier first and then the standard multiplier
         .withColumn("value", col("value") * F.coalesce(col("multiplier"), lit(1)))