sws-spark-dissemination-helper 0.0.107__tar.gz → 0.0.109__tar.gz

{sws_spark_dissemination_helper-0.0.107 → sws_spark_dissemination_helper-0.0.109}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.107
+Version: 0.0.109
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://bitbucket.org/cioapps/sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.107 → sws_spark_dissemination_helper-0.0.109}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.107"
+version = "0.0.109"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3==1.36.18",

{sws_spark_dissemination_helper-0.0.107 → sws_spark_dissemination_helper-0.0.109}/src/sws_spark_dissemination_helper/SWSPostgresSparkReader.py

@@ -291,13 +291,17 @@ class SWSPostgresSparkReader:
         self,
         domain_code: str,
     ) -> DataFrame:
-        return self.read_pg_table_and_check_duplicates_for_domain(
+        df = self.read_pg_table(
             pg_table=DatasetDatatables.MAPPING_CODE_CORRECTION.id,
-            table_name=DatasetDatatables.MAPPING_CODE_CORRECTION.name,
             custom_schema=DatasetDatatables.MAPPING_CODE_CORRECTION.schema,
-            domain_code=domain_code,
-            unique_columns=["old_code"],
         )
+        df.filter(
+            col("mapping_type").isNull() | (col("mapping_type") == lit(""))
+        ).transform(
+            correct_domain_filter, domain=domain_code, unique_columns=["old_code"]
+        )
+
+        return df
 
     def get_domain_code_source_datasets_ids_dest_dataset_id(
         self, dataset_id: str, domain_code: str = None

{sws_spark_dissemination_helper-0.0.107 → sws_spark_dissemination_helper-0.0.109}/src/sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py

@@ -163,6 +163,80 @@ class SWSSilverIcebergSparkHelper:
 
         logging.info(f"Checking time validity for {col_name} of type {col_type}")
 
+        if col_type == "area":
+            df_start_year_correction = self.df_mapping_code_correction.filter(
+                col("var_type") == lit("start_year")
+            )
+            df_end_year_correction = self.df_mapping_code_correction.filter(
+                col("var_type") == lit("end_year")
+            )
+
+            original_col_order = df.columns
+
+            df = (
+                df.alias("d")
+                .join(
+                    F.broadcast(df_start_year_correction).alias("sy"),
+                    on=col(f"d.{col_name}") == col("sy.mapping_type"),
+                    how="left",
+                )
+                .join(
+                    F.broadcast(df_end_year_correction).alias("ey"),
+                    on=col(f"d.{col_name}") == col("ey.mapping_type"),
+                    how="left",
+                )
+                .withColumn("valid_new_start_year", col("sy.new_code").isNotNull())
+                .withColumn("valid_new_end_year", col("ey.new_code").isNotNull())
+                .withColumn(
+                    "note",
+                    F.when(
+                        col("valid_new_start_year"),
+                        F.array_append(
+                            col("d.note"),
+                            F.concat(
+                                col("sy.note"),
+                                lit(" from "),
+                                col("sy.old_code"),
+                                lit(" to "),
+                                col("sy.new_code"),
+                            ),
+                        ),
+                    ).otherwise(col("note")),
+                )
+                .withColumn(
+                    "note",
+                    F.when(
+                        col("valid_new_end_year"),
+                        F.array_append(
+                            col("note"),
+                            F.concat(
+                                col("ey.note"),
+                                lit(" from "),
+                                col("ey.old_code"),
+                                lit(" to "),
+                                col("ey.new_code"),
+                            ),
+                        ),
+                    ).otherwise(col("note")),
+                )
+                .withColumn(
+                    f"{col_name}_start_date",
+                    F.when(
+                        col("valid_new_start_year"), F.to_date(col("sy.new_code"))
+                    ).otherwise(col(f"d.{col_name}_start_date")),
+                )
+                .withColumn(
+                    f"{col_name}_end_date",
+                    F.when(
+                        col("valid_new_end_year"), F.to_date(col("ey.new_code"))
+                    ).otherwise(col(f"d.{col_name}_end_date")),
+                )
+                .selectExpr(
+                    "d.*", "note", f"{col_name}_start_date", f"{col_name}_end_date"
+                )
+                .select(*original_col_order)
+            )
+
         # Iterate through columns and build conditions dynamically
         start_date_condition = col(f"{col_name}_start_date").isNull() | (
             col(f"{col_name}_start_date") <= col(f"{self.time_column}_start_date")