sws-spark-dissemination-helper 0.0.107__py3-none-any.whl → 0.0.109__py3-none-any.whl

sws_spark_dissemination_helper/SWSPostgresSparkReader.py

@@ -291,13 +291,17 @@ class SWSPostgresSparkReader:
         self,
         domain_code: str,
     ) -> DataFrame:
-        return self.read_pg_table_and_check_duplicates_for_domain(
+        df = self.read_pg_table(
             pg_table=DatasetDatatables.MAPPING_CODE_CORRECTION.id,
-            table_name=DatasetDatatables.MAPPING_CODE_CORRECTION.name,
             custom_schema=DatasetDatatables.MAPPING_CODE_CORRECTION.schema,
-            domain_code=domain_code,
-            unique_columns=["old_code"],
         )
+        df.filter(
+            col("mapping_type").isNull() | (col("mapping_type") == lit(""))
+        ).transform(
+            correct_domain_filter, domain=domain_code, unique_columns=["old_code"]
+        )
+
+        return df
 
     def get_domain_code_source_datasets_ids_dest_dataset_id(
         self, dataset_id: str, domain_code: str = None

sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py

@@ -163,6 +163,80 @@ class SWSSilverIcebergSparkHelper:
 
         logging.info(f"Checking time validity for {col_name} of type {col_type}")
 
+        if col_type == "area":
+            df_start_year_correction = self.df_mapping_code_correction.filter(
+                col("var_type") == lit("start_year")
+            )
+            df_end_year_correction = self.df_mapping_code_correction.filter(
+                col("var_type") == lit("end_year")
+            )
+
+            original_col_order = df.columns
+
+            df = (
+                df.alias("d")
+                .join(
+                    F.broadcast(df_start_year_correction).alias("sy"),
+                    on=col(f"d.{col_name}") == col("sy.mapping_type"),
+                    how="left",
+                )
+                .join(
+                    F.broadcast(df_end_year_correction).alias("ey"),
+                    on=col(f"d.{col_name}") == col("ey.mapping_type"),
+                    how="left",
+                )
+                .withColumn("valid_new_start_year", col("sy.new_code").isNotNull())
+                .withColumn("valid_new_end_year", col("ey.new_code").isNotNull())
+                .withColumn(
+                    "note",
+                    F.when(
+                        col("valid_new_start_year"),
+                        F.array_append(
+                            col("d.note"),
+                            F.concat(
+                                col("sy.note"),
+                                lit(" from "),
+                                col("sy.old_code"),
+                                lit(" to "),
+                                col("sy.new_code"),
+                            ),
+                        ),
+                    ).otherwise(col("note")),
+                )
+                .withColumn(
+                    "note",
+                    F.when(
+                        col("valid_new_end_year"),
+                        F.array_append(
+                            col("note"),
+                            F.concat(
+                                col("ey.note"),
+                                lit(" from "),
+                                col("ey.old_code"),
+                                lit(" to "),
+                                col("ey.new_code"),
+                            ),
+                        ),
+                    ).otherwise(col("note")),
+                )
+                .withColumn(
+                    f"{col_name}_start_date",
+                    F.when(
+                        col("valid_new_start_year"), F.to_date(col("sy.new_code"))
+                    ).otherwise(col(f"d.{col_name}_start_date")),
+                )
+                .withColumn(
+                    f"{col_name}_end_date",
+                    F.when(
+                        col("valid_new_end_year"), F.to_date(col("ey.new_code"))
+                    ).otherwise(col(f"d.{col_name}_end_date")),
+                )
+                .selectExpr(
+                    "d.*", "note", f"{col_name}_start_date", f"{col_name}_end_date"
+                )
+                .select(*original_col_order)
+            )
+
         # Iterate through columns and build conditions dynamically
         start_date_condition = col(f"{col_name}_start_date").isNull() | (
             col(f"{col_name}_start_date") <= col(f"{self.time_column}_start_date")

{sws_spark_dissemination_helper-0.0.107.dist-info → sws_spark_dissemination_helper-0.0.109.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.107
+Version: 0.0.109
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://bitbucket.org/cioapps/sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.107.dist-info → sws_spark_dissemination_helper-0.0.109.dist-info}/RECORD

@@ -1,11 +1,11 @@
 sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py,sha256=aCi_rGvPkTDF9Sf5veWPeK_-CyP_GuwXLNnSjJRrd4M,19791
 sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py,sha256=RhgkLoxAT5AZlrj18SMsbONo3FJGZu7gjLU522vG5lg,19319
-sws_spark_dissemination_helper/SWSPostgresSparkReader.py,sha256=ja7AbOfbmC_EXHCJk7UMDzzbA-LRxzPkaaUmuvcihJ8,17449
-sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py,sha256=08xA7h4oVicKOnjEoMrLXpuk9mZ2GTiTPx3C7rCI7pg,22224
+sws_spark_dissemination_helper/SWSPostgresSparkReader.py,sha256=cRLa6KYv7tbGB2_Rpcyvb3ksGu0lX9gpLhnF9nxlxEs,17494
+sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py,sha256=7n9THlgJY9QFZOAFb9BYzBN8cUDr5w1mzsonokzd85k,25203
 sws_spark_dissemination_helper/__init__.py,sha256=Efjoe9V4vGXWVp-DY5P6NbRwIUr_zkZJkDmMi-lf5Bc,262
 sws_spark_dissemination_helper/constants.py,sha256=QKtM8fYB8GWI0yEag4A7NLABRuu1T8qJ156F13YsRzo,11546
 sws_spark_dissemination_helper/utils.py,sha256=6SzrXX0xhvynRyv-vRFDbc6V4UNe_RzKKETZAtefnhg,21341
-sws_spark_dissemination_helper-0.0.107.dist-info/METADATA,sha256=31MxlRsrAZkaUqcwdZPTQfw08YQVzCi1BOljaSQh6UM,2824
-sws_spark_dissemination_helper-0.0.107.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-sws_spark_dissemination_helper-0.0.107.dist-info/licenses/LICENSE,sha256=zFzeb_j_6pXEHwH8Z0OpIkKFJk7vmhZjdem-K0d4zU4,1073
-sws_spark_dissemination_helper-0.0.107.dist-info/RECORD,,
+sws_spark_dissemination_helper-0.0.109.dist-info/METADATA,sha256=cE81bRoInx7Ezl6GEctwvOyqYUVBQIJCM_iuiWoWYdQ,2824
+sws_spark_dissemination_helper-0.0.109.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+sws_spark_dissemination_helper-0.0.109.dist-info/licenses/LICENSE,sha256=zFzeb_j_6pXEHwH8Z0OpIkKFJk7vmhZjdem-K0d4zU4,1073
+sws_spark_dissemination_helper-0.0.109.dist-info/RECORD,,