sws-spark-dissemination-helper 0.0.111__tar.gz → 0.0.113__tar.gz

{sws_spark_dissemination_helper-0.0.111 → sws_spark_dissemination_helper-0.0.113}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.111
+Version: 0.0.113
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://bitbucket.org/cioapps/sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.111 → sws_spark_dissemination_helper-0.0.113}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.111"
+version = "0.0.113"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3==1.36.18",

{sws_spark_dissemination_helper-0.0.111 → sws_spark_dissemination_helper-0.0.113}/src/sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py

@@ -10,7 +10,7 @@ from pyspark.sql.window import Window
 from sws_api_client import Tags
 from sws_api_client.tags import BaseDisseminatedTagTable, TableLayer, TableType
 
-from .constants import IcebergDatabases, IcebergTables
+from .constants import IcebergDatabases, IcebergTables, DatasetDatatables
 from .SWSPostgresSparkReader import SWSPostgresSparkReader
 from .utils import (
     get_or_create_tag,
@@ -164,6 +164,9 @@ class SWSSilverIcebergSparkHelper:
         logging.info(f"Checking time validity for {col_name} of type {col_type}")
 
         if col_type == "area":
+            logging.info(
+                f'Changing start and end year according to "{DatasetDatatables.MAPPING_CODE_CORRECTION.name}"'
+            )
             df_start_year_correction = self.df_mapping_code_correction.filter(
                 col("var_type") == lit("start_year")
             )
@@ -232,10 +235,11 @@ class SWSSilverIcebergSparkHelper:
                 .withColumn(
                     f"new_{col_name}_end_date",
                     F.when(
-                        col("valid_new_end_year"), F.to_date(col("ey.new_code"))
+                        col("valid_new_end_year"),
+                        F.to_date(F.concat(col("ey.new_code"), lit("-12-31"))),
                     ).otherwise(col(f"d.{col_name}_end_date")),
                 )
-                .selectExpr(
+                .select(
                     *cols_to_select,
                     col("new_note").alias("note"),
                     col(f"new_{col_name}_start_date").alias(f"{col_name}_start_date"),