sws-spark-dissemination-helper 0.0.107__py3-none-any.whl → 0.0.108__py3-none-any.whl

sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py

@@ -163,6 +163,80 @@ class SWSSilverIcebergSparkHelper:
 
         logging.info(f"Checking time validity for {col_name} of type {col_type}")
 
+        if col_type == "area":
+            df_start_year_correction = self.df_mapping_code_correction.filter(
+                col("var_type") == lit("start_year")
+            )
+            df_end_year_correction = self.df_mapping_code_correction.filter(
+                col("var_type") == lit("end_year")
+            )
+
+            original_col_order = df.columns
+
+            df = (
+                df.alias("d")
+                .join(
+                    F.broadcast(df_start_year_correction).alias("sy"),
+                    on=col(f"d.{col_name}") == col("sy.mapping_type"),
+                    how="left",
+                )
+                .join(
+                    F.broadcast(df_end_year_correction).alias("ey"),
+                    on=col(f"d.{col_name}") == col("ey.mapping_type"),
+                    how="left",
+                )
+                .withColumn("valid_new_start_year", col("sy.new_code").isNotNull())
+                .withColumn("valid_new_end_year", col("ey.new_code").isNotNull())
+                .withColumn(
+                    "note",
+                    F.when(
+                        col("valid_new_start_year"),
+                        F.array_append(
+                            col("d.note"),
+                            F.concat(
+                                col("sy.note"),
+                                lit(" from "),
+                                col("sy.old_code"),
+                                lit(" to "),
+                                col("sy.new_code"),
+                            ),
+                        ),
+                    ).otherwise(col("note")),
+                )
+                .withColumn(
+                    "note",
+                    F.when(
+                        col("valid_new_end_year"),
+                        F.array_append(
+                            col("note"),
+                            F.concat(
+                                col("ey.note"),
+                                lit(" from "),
+                                col("ey.old_code"),
+                                lit(" to "),
+                                col("ey.new_code"),
+                            ),
+                        ),
+                    ).otherwise(col("note")),
+                )
+                .withColumn(
+                    f"{col_name}_start_date",
+                    F.when(
+                        col("valid_new_start_year"), F.to_date(col("sy.new_code"))
+                    ).otherwise(col(f"d.{col_name}_start_date")),
+                )
+                .withColumn(
+                    f"{col_name}_end_date",
+                    F.when(
+                        col("valid_new_end_year"), F.to_date(col("ey.new_code"))
+                    ).otherwise(col(f"d.{col_name}_end_date")),
+                )
+                .selectExpr(
+                    "d.*", "note", f"{col_name}_start_date", f"{col_name}_end_date"
+                )
+                .select(*original_col_order)
+            )
+
         # Iterate through columns and build conditions dynamically
         start_date_condition = col(f"{col_name}_start_date").isNull() | (
             col(f"{col_name}_start_date") <= col(f"{self.time_column}_start_date")

{sws_spark_dissemination_helper-0.0.107.dist-info → sws_spark_dissemination_helper-0.0.108.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.107
+Version: 0.0.108
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://bitbucket.org/cioapps/sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.107.dist-info → sws_spark_dissemination_helper-0.0.108.dist-info}/RECORD

@@ -1,11 +1,11 @@
 sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py,sha256=aCi_rGvPkTDF9Sf5veWPeK_-CyP_GuwXLNnSjJRrd4M,19791
 sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py,sha256=RhgkLoxAT5AZlrj18SMsbONo3FJGZu7gjLU522vG5lg,19319
 sws_spark_dissemination_helper/SWSPostgresSparkReader.py,sha256=ja7AbOfbmC_EXHCJk7UMDzzbA-LRxzPkaaUmuvcihJ8,17449
-sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py,sha256=08xA7h4oVicKOnjEoMrLXpuk9mZ2GTiTPx3C7rCI7pg,22224
+sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py,sha256=7n9THlgJY9QFZOAFb9BYzBN8cUDr5w1mzsonokzd85k,25203
 sws_spark_dissemination_helper/__init__.py,sha256=Efjoe9V4vGXWVp-DY5P6NbRwIUr_zkZJkDmMi-lf5Bc,262
 sws_spark_dissemination_helper/constants.py,sha256=QKtM8fYB8GWI0yEag4A7NLABRuu1T8qJ156F13YsRzo,11546
 sws_spark_dissemination_helper/utils.py,sha256=6SzrXX0xhvynRyv-vRFDbc6V4UNe_RzKKETZAtefnhg,21341
-sws_spark_dissemination_helper-0.0.107.dist-info/METADATA,sha256=31MxlRsrAZkaUqcwdZPTQfw08YQVzCi1BOljaSQh6UM,2824
-sws_spark_dissemination_helper-0.0.107.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-sws_spark_dissemination_helper-0.0.107.dist-info/licenses/LICENSE,sha256=zFzeb_j_6pXEHwH8Z0OpIkKFJk7vmhZjdem-K0d4zU4,1073
-sws_spark_dissemination_helper-0.0.107.dist-info/RECORD,,
+sws_spark_dissemination_helper-0.0.108.dist-info/METADATA,sha256=_5o4LZWS2h6v02vjHuqHL8AlmRc74QPXS7zhQ-3khsg,2824
+sws_spark_dissemination_helper-0.0.108.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+sws_spark_dissemination_helper-0.0.108.dist-info/licenses/LICENSE,sha256=zFzeb_j_6pXEHwH8Z0OpIkKFJk7vmhZjdem-K0d4zU4,1073
+sws_spark_dissemination_helper-0.0.108.dist-info/RECORD,,