sws-spark-dissemination-helper 0.0.60__py3-none-any.whl → 0.0.171__py3-none-any.whl

sws_spark_dissemination_helper/utils.py

@@ -4,12 +4,13 @@ from typing import List
 
 import boto3
 import pyspark.sql.functions as F
-from pyspark.sql import DataFrame, SparkSession, Column
+from pyspark.sql import Column, DataFrame, SparkSession
 from pyspark.sql.functions import col, lit
+from pyspark.sql.window import Window
 from sws_api_client import Tags
-from sws_api_client.tags import DisseminatedTag, BaseDisseminatedTagTable
+from sws_api_client.tags import BaseDisseminatedTagTable, DisseminatedTag
 
-from .constants import DomainFilters, DatasetDatatables
+from .constants import DatasetDatatables, DomainFilters
 
 
 def get_spark() -> SparkSession:
@@ -272,7 +273,7 @@ def check_sdmx_col_names_mappings(
     return df_mapping_sdmx_column_names_unique
 
 
-def map_codes(
+def map_codes_and_remove_null_duplicates(
     df: DataFrame,
     df_mapping: DataFrame,
     domain_code: str,
@@ -280,98 +281,162 @@ def map_codes(
     col_type: str,
     src_column: str,
     dest_column: str,
+    dimension_columns: List[str],
+    flag_columns: List[str],
 ) -> DataFrame:
-    return (
+
+    lower_col_name = col_name.lower()
+    lower_flag_columns = [column.lower() for column in flag_columns]
+    lower_dimension_columns = [column.lower() for column in dimension_columns]
+
+    # Define partitioning columns
+    if lower_col_name in lower_flag_columns:
+        partition_columns = dimension_columns
+    else:
+        partition_columns = [
+            column for column in lower_dimension_columns if column != lower_col_name
+        ] + ["partition_column"]
+
+    partitioning_window = Window.partitionBy(*partition_columns)
+
+    standard_mapping_df = df_mapping.filter(
+        (col("domain").isNull() | (col("domain") == lit("")))
+        & (col("var_type") == lit(col_type))
+        & (col("mapping_type").isNull() | (col("mapping_type") == lit("")))
+    )
+    domain_mapping_df = df_mapping.filter(
+        (col("domain") == lit(domain_code))
+        & (col("var_type") == lit(col_type))
+        & (col("mapping_type").isNull() | (col("mapping_type") == lit("")))
+    )
+
+    count_all = df.count()
+
+    df_no_nulls = (
         df.alias("d")
         # Join the data with the standard mapping for the specific dimension
         .join(
-            F.broadcast(
-                df_mapping.filter(
-                    (col("domain").isNull() | (col("domain") == lit("")))
-                    & (col("var_type") == lit(col_type))
-                    & (
-                        col("mapping_type").isNull()
-                        | (col("mapping_type").isNull() == lit(""))
-                    )
-                )
-            ).alias("m_standard"),
+            F.broadcast(standard_mapping_df).alias("m_standard"),
             col(f"d.{col_name}") == col(f"m_standard.{src_column}"),
             "left",
         )
         # Join the data with the domain specific mapping for the specific dimension
         .join(
-            F.broadcast(
-                df_mapping.filter(
-                    (col("domain") == lit(domain_code))
-                    & (col("var_type") == lit(col_type))
-                    & (
-                        col("mapping_type").isNull()
-                        | (col("mapping_type").isNull() == lit(""))
-                    )
-                )
-            ).alias("m_domain"),
+            F.broadcast(domain_mapping_df).alias("m_domain"),
             col(f"d.{col_name}") == col(f"m_domain.{src_column}"),
             "left",
         )
-        # Select only the columns we are interested in (this step is optional but recommended for debugging)
         .select(
             "d.*",
-            col(f"m_standard.{dest_column}").alias(f"standard_{dest_column}"),
-            col("m_standard.delete").alias("standard_delete"),
-            col("m_standard.multiplier").alias("standard_multiplier"),
-            col(f"m_domain.{dest_column}").alias(f"domain_specific_{dest_column}"),
-            col("m_domain.delete").alias("domain_specific_delete"),
-            col("m_domain.multiplier").alias("domain_specific_multiplier"),
-        )
-        # Filter out records to delete
-        .filter(
-            # Evaluate first the domain specific flag
+            # Evaluate the domain specific rule first and then the general rule
+            F.coalesce(
+                col(f"m_domain.{dest_column}"), col(f"m_standard.{dest_column}")
+            ).alias("new_dim_code"),
+            F.coalesce(
+                col("m_domain.delete"),
+                col("m_standard.delete"),
+                lit(False),
+            ).alias("delete"),
+            F.coalesce(col("m_standard.multiplier"), col("m_domain.multiplier")).alias(
+                "multiplier"
+            ),
+        )
+        .withColumn("partition_column", F.coalesce(col("new_dim_code"), col(col_name)))
+        .withColumn("count_obs_per_point", F.count(lit(1)).over(partitioning_window))
+        .withColumn("is_duplicate", col("count_obs_per_point") > lit(1))
+        # Filter out all the rows that are duplicates with null value
+        .filter(~(col("is_duplicate") & col("value").isNull()))
+    )
+
+    count_no_null_dupes = df_no_nulls.count()
+    null_dupes_removed = count_all - count_no_null_dupes
+
+    logging.info(f"{null_dupes_removed} duplicates with null value removed")
+
+    df_mapped = (
+        df_no_nulls
+        # Count again the observations per coordinate after removing the null duplicates
+        .withColumn("count_obs_per_point", F.count(lit(1)).over(partitioning_window))
+        .withColumn("is_duplicate", col("count_obs_per_point") > lit(1))
+        # Update the diss_flag to false for records to delete
+        .withColumn(
+            "diss_flag", F.when(col("delete"), lit(False)).otherwise(col("diss_flag"))
+        )
+        .withColumn(
+            "note",
+            F.when(
+                col("delete"),
+                F.array_append(
+                    col("note"),
+                    lit(
+                        f"The observation is not disseminated according to the Mapping - Code correction table"
+                    ),
+                ),
+            ).otherwise(col("note")),
+        )
+        # Add mapping message to notes
+        .withColumn(
+            "note",
             F.when(
-                col("domain_specific_delete").isNotNull(),
-                ~col("domain_specific_delete"),
-            )
-            # Then evaluate the general flag
-            .when(
-                col("standard_delete").isNotNull(), ~col("standard_delete")
-            ).otherwise(lit(True))
+                ~col("is_duplicate")
+                & col("new_dim_code").isNotNull()
+                & (col("new_dim_code") != lit("")),
+                F.array_append(
+                    col("note"),
+                    F.concat(
+                        lit(f"Dimension {col_name} code was changed from "),
+                        col(col_name),
+                        lit(" to "),
+                        col("new_dim_code"),
+                    ),
+                ),
+            ).otherwise(col("note")),
         )
         .withColumn(
             col_name,
-            # Evaluate first the domain specific mapping
             F.when(
-                col(f"domain_specific_{dest_column}").isNotNull(),
-                col(f"domain_specific_{dest_column}"),
-            )
-            # Then evaluate the general mapping
-            .when(
-                col(f"standard_{dest_column}").isNotNull(),
-                col(f"standard_{dest_column}"),
+                ~col("is_duplicate"),
+                F.coalesce(col("new_dim_code"), col(col_name)),
             ).otherwise(col(col_name)),
         )
         .withColumn(
-            "value",
-            # Multiply first by the domain specific multiplier
+            "diss_flag",
+            F.when(
+                col("is_duplicate")
+                & col("new_dim_code").isNotNull()
+                & (col("new_dim_code") != lit("")),
+                lit(False),
+            ).otherwise(col("diss_flag")),
+        )
+        .withColumn(
+            "note",
             F.when(
-                col("domain_specific_multiplier").isNotNull(),
-                col("value") * col("domain_specific_multiplier"),
-            )
-            # Then multiply by the general multiplier
-            .when(
-                col(f"standard_{dest_column}").isNotNull(),
-                col("value") * col("standard_multiplier"),
-            ).otherwise(col("value")),
+                col("is_duplicate")
+                & col("new_dim_code").isNotNull()
+                & (col("new_dim_code") != lit("")),
+                F.array_append(
+                    col("note"),
+                    lit(
+                        f"The code correction was not applied to avoid observation duplications"
+                    ),
+                ),
+            ).otherwise(col("note")),
         )
+        # Check the domain specific multiplier first and then the standard multiplier
+        .withColumn("value", col("value") * F.coalesce(col("multiplier"), lit(1)))
         # Remove the columns that were not in the original dataset
         .drop(
-            f"standard_{dest_column}",
-            "standard_delete",
-            "standard_multiplier",
-            f"domain_specific_{dest_column}",
-            "domain_specific_delete",
-            "domain_specific_multiplier",
+            "new_dim_code",
+            "delete",
+            "multiplier",
+            "partition_column",
+            "count_obs_per_point",
+            "is_duplicate",
         )
     )
 
+    return df_mapped
+
 
 def apply_code_correction(
     df: DataFrame,
@@ -381,7 +446,7 @@ def apply_code_correction(
     col_type: str,
 ) -> DataFrame:
     logging.info(f"correcting codes for column {col_name} of type {col_type}")
-    return map_codes(
+    return map_codes_and_remove_null_duplicates(
         df,
         df_mapping_code_correction,
         domain_code,

{sws_spark_dissemination_helper-0.0.60.dist-info → sws_spark_dissemination_helper-0.0.171.dist-info}/METADATA

@@ -1,8 +1,8 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.60
+Version: 0.0.171
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
-Project-URL: Repository, https://bitbucket.org/cioapps/sws-it-python-spark-dissemination-helper
+Project-URL: Repository, https://github.com/un-fao/fao-sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>
 License: MIT License
         
@@ -29,26 +29,30 @@ License-File: LICENSE
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
-Requires-Python: >=3.8
+Requires-Python: >=3.9
 Requires-Dist: annotated-types==0.7.0
-Requires-Dist: boto3==1.34.147
-Requires-Dist: botocore==1.34.147
-Requires-Dist: certifi==2024.7.4
-Requires-Dist: charset-normalizer==3.3.2
-Requires-Dist: idna==3.7
+Requires-Dist: boto3>=1.40.75
+Requires-Dist: botocore>=1.40.75
+Requires-Dist: certifi==2025.1.31
+Requires-Dist: charset-normalizer==3.4.1
+Requires-Dist: idna>=3.10
 Requires-Dist: jmespath==1.0.1
+Requires-Dist: numpy==2.0.2
+Requires-Dist: pandas==2.3.3
 Requires-Dist: py4j==0.10.9.7
-Requires-Dist: pydantic-core==2.20.1
-Requires-Dist: pydantic==2.8.2
-Requires-Dist: pyspark==3.5.1
+Requires-Dist: pydantic-core==2.27.2
+Requires-Dist: pydantic==2.10.6
+Requires-Dist: pyspark==3.5.4
 Requires-Dist: python-dateutil==2.9.0.post0
-Requires-Dist: python-dotenv==1.0.1
+Requires-Dist: python-dotenv==0.19.2
+Requires-Dist: pytz==2025.2
 Requires-Dist: requests==2.32.3
-Requires-Dist: s3transfer==0.10.2
-Requires-Dist: six==1.16.0
-Requires-Dist: sws-api-client==1.0.7b0
-Requires-Dist: typing-extensions==4.12.2
-Requires-Dist: urllib3==1.26.19
+Requires-Dist: s3transfer>=0.11.2
+Requires-Dist: six==1.17.0
+Requires-Dist: sws-api-client==2.3.0
+Requires-Dist: typing-extensions>=4.12.2
+Requires-Dist: tzdata==2025.2
+Requires-Dist: urllib3==1.26.20
 Description-Content-Type: text/markdown
 
 # Upload a new version

sws_spark_dissemination_helper-0.0.171.dist-info/RECORD

@@ -0,0 +1,13 @@
+sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py,sha256=N0eQ2LXtpPeZQCWYi85sMLmpXRzLA2erECiba8tqOAY,29595
+sws_spark_dissemination_helper/SWSDatatablesExportHelper.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sws_spark_dissemination_helper/SWSEasyIcebergSparkHelper.py,sha256=csqKyYglBkJSBvEkEa1_keHarZZAIJHaV0d64gGJy98,26379
+sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py,sha256=0dxbVkrhdaASapEffF5PFcgKwAMyJoWBxzgymjZ4JyY,25049
+sws_spark_dissemination_helper/SWSPostgresSparkReader.py,sha256=KpG8gp8Ai9pHDiKhUOTcXWxxmFGeKEE3XKlI_Y-SveU,18453
+sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py,sha256=qioLv3SlJEfk0LzTiwfXRtZXVImPOJUeh9k1XwHC-pA,26225
+sws_spark_dissemination_helper/__init__.py,sha256=42TPbk7KxAud_qY3Sr_F4F7VjyofUlxEJkUXAFQsjRo,327
+sws_spark_dissemination_helper/constants.py,sha256=vQmalAqInwPAybgJOfYx99jn47KsKp8jeD8eqmjw-Rs,13471
+sws_spark_dissemination_helper/utils.py,sha256=G7lQqNRrvqZpgm9WmddD7fWsI8IVn09x1p3cV3458EA,21963
+sws_spark_dissemination_helper-0.0.171.dist-info/METADATA,sha256=W4qkQISSzekzXhpmNhlNMfJEmaQlscu3hQTs4Vavawg,2824
+sws_spark_dissemination_helper-0.0.171.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+sws_spark_dissemination_helper-0.0.171.dist-info/licenses/LICENSE,sha256=zFzeb_j_6pXEHwH8Z0OpIkKFJk7vmhZjdem-K0d4zU4,1073
+sws_spark_dissemination_helper-0.0.171.dist-info/RECORD,,

sws_spark_dissemination_helper-0.0.60.dist-info/RECORD

@@ -1,11 +0,0 @@
-sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py,sha256=tyC3e2LNBes9J2UFR-j7bDlvEffeI0YsiYlMvk0wPxA,16382
-sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py,sha256=43ZOaDtFdnuYHL9oMSh64F3YUiDDWbVHS5iuhrsE1C4,26074
-sws_spark_dissemination_helper/SWSPostgresSparkReader.py,sha256=wXSz4-SbIcfVfDsN5gsbg6ul5GvVoX59VkfjAmTCToo,14935
-sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py,sha256=ZTpjkejKJpl6kue8DI1FVEJB-M7TlyUgrjXqF7GUEws,21978
-sws_spark_dissemination_helper/__init__.py,sha256=Efjoe9V4vGXWVp-DY5P6NbRwIUr_zkZJkDmMi-lf5Bc,262
-sws_spark_dissemination_helper/constants.py,sha256=KGfuudVovMxgzCIowe7L9gqDbHjbngqzhd3Zgowo5yk,11229
-sws_spark_dissemination_helper/utils.py,sha256=MLiQV1I-HJtc9gHHWn1mPiYCsfI_7bCMPv9GUDY6kO0,19768
-sws_spark_dissemination_helper-0.0.60.dist-info/METADATA,sha256=eNuM3tOPZQ1b0akwWtVqT43UrSlE7YFcTG8lSOHloQQ,2708
-sws_spark_dissemination_helper-0.0.60.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-sws_spark_dissemination_helper-0.0.60.dist-info/licenses/LICENSE,sha256=zFzeb_j_6pXEHwH8Z0OpIkKFJk7vmhZjdem-K0d4zU4,1073
-sws_spark_dissemination_helper-0.0.60.dist-info/RECORD,,