sws-spark-dissemination-helper 0.0.112__tar.gz → 0.0.114__tar.gz

{sws_spark_dissemination_helper-0.0.112 → sws_spark_dissemination_helper-0.0.114}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.112
+Version: 0.0.114
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://bitbucket.org/cioapps/sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>
@@ -49,7 +49,7 @@ Requires-Dist: pytz==2025.1
 Requires-Dist: requests==2.32.3
 Requires-Dist: s3transfer==0.11.2
 Requires-Dist: six==1.17.0
-Requires-Dist: sws-api-client==1.4.5
+Requires-Dist: sws-api-client>=1.5.0
 Requires-Dist: typing-extensions==4.12.2
 Requires-Dist: tzdata==2025.1
 Requires-Dist: urllib3==1.26.20

{sws_spark_dissemination_helper-0.0.112 → sws_spark_dissemination_helper-0.0.114}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.112"
+version = "0.0.114"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3==1.36.18",
@@ -25,7 +25,7 @@ dependencies = [
     "requests==2.32.3",
     "s3transfer==0.11.2",
     "six==1.17.0",
-    "sws_api_client==1.4.5",
+    "sws_api_client>=1.5.0",
     "typing_extensions==4.12.2",
     "tzdata==2025.1",
     "urllib3==1.26.20"

{sws_spark_dissemination_helper-0.0.112 → sws_spark_dissemination_helper-0.0.114}/requirements.txt

@@ -17,7 +17,7 @@ pytz==2025.1
 requests==2.32.3
 s3transfer==0.11.2
 six==1.17.0
-sws_api_client==1.4.5
+sws_api_client==1.5.0
 typing_extensions==4.12.2
 tzdata==2025.1
 urllib3==1.26.20

{sws_spark_dissemination_helper-0.0.112 → sws_spark_dissemination_helper-0.0.114}/src/sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py

@@ -10,7 +10,7 @@ from pyspark.sql.window import Window
 from sws_api_client import Tags
 from sws_api_client.tags import BaseDisseminatedTagTable, TableLayer, TableType
 
-from .constants import IcebergDatabases, IcebergTables
+from .constants import IcebergDatabases, IcebergTables, DatasetDatatables
 from .SWSPostgresSparkReader import SWSPostgresSparkReader
 from .utils import (
     get_or_create_tag,
@@ -164,6 +164,9 @@ class SWSSilverIcebergSparkHelper:
         logging.info(f"Checking time validity for {col_name} of type {col_type}")
 
         if col_type == "area":
+            logging.info(
+                f'Changing start and end year according to "{DatasetDatatables.MAPPING_CODE_CORRECTION.name}"'
+            )
             df_start_year_correction = self.df_mapping_code_correction.filter(
                 col("var_type") == lit("start_year")
             )
@@ -232,7 +235,8 @@ class SWSSilverIcebergSparkHelper:
                 .withColumn(
                     f"new_{col_name}_end_date",
                     F.when(
-                        col("valid_new_end_year"), F.to_date(col("ey.new_code"))
+                        col("valid_new_end_year"),
+                        F.to_date(F.concat(col("ey.new_code"), lit("-12-31"))),
                     ).otherwise(col(f"d.{col_name}_end_date")),
                 )
                 .select(