sws-spark-dissemination-helper 0.0.108__tar.gz → 0.0.109__tar.gz

{sws_spark_dissemination_helper-0.0.108 → sws_spark_dissemination_helper-0.0.109}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.108
+Version: 0.0.109
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://bitbucket.org/cioapps/sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>

{sws_spark_dissemination_helper-0.0.108 → sws_spark_dissemination_helper-0.0.109}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.108"
+version = "0.0.109"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3==1.36.18",

{sws_spark_dissemination_helper-0.0.108 → sws_spark_dissemination_helper-0.0.109}/src/sws_spark_dissemination_helper/SWSPostgresSparkReader.py

@@ -291,13 +291,17 @@ class SWSPostgresSparkReader:
         self,
         domain_code: str,
     ) -> DataFrame:
-        return self.read_pg_table_and_check_duplicates_for_domain(
+        df = self.read_pg_table(
             pg_table=DatasetDatatables.MAPPING_CODE_CORRECTION.id,
-            table_name=DatasetDatatables.MAPPING_CODE_CORRECTION.name,
             custom_schema=DatasetDatatables.MAPPING_CODE_CORRECTION.schema,
-            domain_code=domain_code,
-            unique_columns=["old_code"],
         )
+        df.filter(
+            col("mapping_type").isNull() | (col("mapping_type") == lit(""))
+        ).transform(
+            correct_domain_filter, domain=domain_code, unique_columns=["old_code"]
+        )
+
+        return df
 
     def get_domain_code_source_datasets_ids_dest_dataset_id(
         self, dataset_id: str, domain_code: str = None