sws-spark-dissemination-helper 0.0.84__tar.gz → 0.0.85__tar.gz

{sws_spark_dissemination_helper-0.0.84 → sws_spark_dissemination_helper-0.0.85}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.84
+Version: 0.0.85
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
 Project-URL: Repository, https://bitbucket.org/cioapps/sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>
@@ -49,7 +49,7 @@ Requires-Dist: pytz==2025.1
 Requires-Dist: requests==2.32.3
 Requires-Dist: s3transfer==0.11.2
 Requires-Dist: six==1.17.0
-Requires-Dist: sws-api-client==1.4.4
+Requires-Dist: sws-api-client==1.4.5
 Requires-Dist: typing-extensions==4.12.2
 Requires-Dist: tzdata==2025.1
 Requires-Dist: urllib3==1.26.20

sws_spark_dissemination_helper-0.0.85/old_requirements.txt

@@ -0,0 +1,23 @@
+annotated-types==0.7.0
+boto3==1.36.18
+botocore==1.36.18
+certifi==2025.1.31
+charset-normalizer==3.4.1
+idna==3.10
+jmespath==1.0.1
+numpy==2.0.2
+pandas==2.2.3
+py4j==0.10.9.7
+pydantic==2.10.6
+pydantic_core==2.27.2
+pyspark==3.5.4
+python-dateutil==2.9.0.post0
+python-dotenv==0.19.2
+pytz==2025.1
+requests==2.32.3
+s3transfer==0.11.2
+six==1.17.0
+sws_api_client==1.4.4
+typing_extensions==4.12.2
+tzdata==2025.1
+urllib3==1.26.20

{sws_spark_dissemination_helper-0.0.84 → sws_spark_dissemination_helper-0.0.85}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sws-spark-dissemination-helper"
-version = "0.0.84"
+version = "0.0.85"
 dependencies = [
     "annotated-types==0.7.0",
     "boto3==1.36.18",
@@ -25,7 +25,7 @@ dependencies = [
     "requests==2.32.3",
     "s3transfer==0.11.2",
     "six==1.17.0",
-    "sws_api_client==1.4.4",
+    "sws_api_client==1.4.5",
     "typing_extensions==4.12.2",
     "tzdata==2025.1",
     "urllib3==1.26.20"

sws_spark_dissemination_helper-0.0.85/requirements.txt

@@ -0,0 +1,23 @@
+annotated-types==0.7.0
+boto3==1.36.18
+botocore==1.36.18
+certifi==2025.1.31
+charset-normalizer==3.4.1
+idna==3.10
+jmespath==1.0.1
+numpy==2.0.2
+pandas==2.2.3
+py4j==0.10.9.7
+pydantic==2.10.6
+pydantic_core==2.27.2
+pyspark==3.5.4
+python-dateutil==2.9.0.post0
+python-dotenv==0.19.2
+pytz==2025.1
+requests==2.32.3
+s3transfer==0.11.2
+six==1.17.0
+sws_api_client==1.4.5
+typing_extensions==4.12.2
+tzdata==2025.1
+urllib3==1.26.20

{sws_spark_dissemination_helper-0.0.84 → sws_spark_dissemination_helper-0.0.85}/src/sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py

@@ -11,7 +11,6 @@ from sws_api_client.tags import BaseDisseminatedTagTable, TableLayer, TableType
 from .constants import IcebergDatabases, IcebergTables
 from .SWSPostgresSparkReader import SWSPostgresSparkReader
 from .utils import (
-    col_is_null_or_empty,
     get_or_create_tag,
     save_cache_csv,
     upsert_disseminated_table,
@@ -47,11 +46,6 @@ class SWSGoldIcebergSparkHelper:
                 self.flag_columns,
             ) = self._get_dim_time_flag_columns()
 
-            self.cols_to_keep_sdmx = (
-                self.dim_columns_w_time
-                + ["unit_of_measure", "unit_of_measure_multiplier", "value"]
-                + self.flag_columns
-            )
             self.cols_to_keep_sws = (
                 self.dim_columns_w_time + ["value"] + self.flag_columns
             )
@@ -72,42 +66,6 @@ class SWSGoldIcebergSparkHelper:
                 if col_name in self.dim_columns
             }
 
-            (
-                self.df_mapping_sdmx_codes,
-                self.df_mapping_sdmx_uom,
-                self.df_mapping_sdmx_col_names,
-            ) = sws_postgres_spark_reader.import_sdmx_mapping_datatables(
-                self.domain_code
-            )
-
-            self._check_column_mappings(self.df_mapping_sdmx_col_names)
-
-    def _check_column_mappings(
-        self,
-        df_mapping_sdmx_col_names: DataFrame,
-    ) -> DataFrame:
-        cols_to_keep_set = set(self.cols_to_keep_sdmx)
-        mapping_sdmx_col_names_internal_set = {
-            row[0]
-            for row in df_mapping_sdmx_col_names.filter(
-                col("internal_name").isNotNull() & (col("internal_name") != lit(""))
-            )
-            .select("internal_name")
-            .collect()
-        }
-
-        if not (cols_to_keep_set <= mapping_sdmx_col_names_internal_set):
-            missing_mappings = cols_to_keep_set - mapping_sdmx_col_names_internal_set
-
-            message = 'The mappings in the table "Mapping - SDMX columns names" are not correct'
-
-            if len(missing_mappings) > 0:
-                message += (
-                    f"\nThe following column mappings are missing: {missing_mappings}"
-                )
-
-            raise ValueError(message)
-
     def _get_dim_time_flag_columns(self) -> Tuple[List[str], List[str], str, List[str]]:
         """Extract the dimension columns with time, without time, the time column and the flag columns names."""
         dim_columns_w_time = [
@@ -128,292 +86,9 @@ class SWSGoldIcebergSparkHelper:
     def apply_diss_flag_filter(self, df: DataFrame) -> DataFrame:
         return df.filter(col("diss_flag"))
 
-    # TODO implement the delete flag
-    def apply_uom_mapping(
-        self,
-        df: DataFrame,
-    ) -> DataFrame:
-        logging.info("mapping unit of measure for dissemination")
-
-        df = df.withColumn(
-            "official_sws_uom",
-            F.when(
-                col_is_null_or_empty("unit_of_measure_base_unit"),
-                col("unit_of_measure"),
-            ).otherwise(col("unit_of_measure_base_unit")),
-        ).withColumn(
-            "official_sws_multiplier",
-            F.coalesce(F.log10(col("unit_of_measure_multiplier")), lit(0)).cast("int"),
-        )
-
-        delete_df_uom_mapping = self.df_mapping_sdmx_uom.filter(
-            col("delete")
-            & col_is_null_or_empty("sdmx_code")
-            & col("sdmx_multiplier").isNull()
-            & col("value_multiplier").isNull()
-        )
-
-        generic_df_uom_mapping = self.df_mapping_sdmx_uom.filter(
-            ~col("delete")
-            & col("sws_multiplier").isNull()
-            & col("sdmx_multiplier").isNull()
-            & (col("value_multiplier") == lit(0))
-        )
-
-        specific_df_uom_mapping = self.df_mapping_sdmx_uom.filter(
-            ~col("delete")
-            & col("sws_multiplier").isNotNull()
-            & col("sdmx_multiplier").isNotNull()
-        )
-
-        # Apply generic uom mapping
-        df = (
-            df.alias("d")
-            .join(
-                generic_df_uom_mapping.alias("m"),
-                col("d.official_sws_uom") == col("m.sws_code"),
-                "left",
-            )
-            .select("d.*", col("sdmx_code").alias("generic_sdmx_uom"))
-        )
-
-        # Apply specific uom mapping
-        df = (
-            df.alias("d")
-            .join(
-                specific_df_uom_mapping.alias("m"),
-                (col("d.official_sws_uom") == col("m.sws_code"))
-                & (col("d.official_sws_multiplier") == col("m.sws_multiplier")),
-                "left",
-            )
-            .select(
-                "d.*",
-                col("sdmx_code").alias("specific_sdmx_uom"),
-                col("sdmx_multiplier").alias("specific_sdmx_multiplier"),
-                (col("value") * F.pow(lit(10), col("value_multiplier"))).alias(
-                    "specific_sdmx_value"
-                ),
-            )
-        )
-
-        # Select the official values according to descending specificity
-        df = (
-            df.withColumn(
-                "unit_of_measure",
-                F.coalesce(
-                    col("specific_sdmx_uom"),
-                    col("generic_sdmx_uom"),
-                    col("official_sws_uom"),
-                ),
-            )
-            .withColumn(
-                "unit_of_measure_multiplier",
-                F.coalesce(
-                    col("specific_sdmx_multiplier"), col("official_sws_multiplier")
-                ),
-            )
-            .withColumn(
-                "value",
-                F.coalesce(col("specific_sdmx_value"), col("value")),
-            )
-            # Remove the columns that were not in the original dataset
-            .drop(
-                col("specific_sdmx_uom"),
-                col("specific_sdmx_multiplier"),
-                col("specific_sdmx_value"),
-                col("generic_sdmx_uom"),
-                col("official_sws_uom"),
-                col("official_sws_multiplier"),
-            )
-        )
-
-        return df
-
-    def keep_dim_uom_val_attr_columns(self, df: DataFrame):
-        return df.select(*self.cols_to_keep_sdmx)
-
     def keep_dim_val_attr_columns(self, df: DataFrame):
         return df.select(*self.cols_to_keep_sws)
 
-    def _apply_sdmx_dimension_codes_mapping_single(
-        self,
-        df: DataFrame,
-        dimension_name: str,
-        dimension_type: str,
-    ) -> DataFrame:
-        logging.info(
-            f"mapping column {dimension_name} of type {dimension_type} for dissemination"
-        )
-        return (
-            df.alias("d")
-            # Join the data with the standard mapping for the specific dimension
-            .join(
-                F.broadcast(
-                    self.df_mapping_sdmx_codes.filter(
-                        (col("domain").isNull() | (col("domain") == lit("")))
-                        & (col("var_type") == lit(dimension_type))
-                        & (
-                            col("mapping_type").isNull()
-                            | (col("mapping_type") == lit(""))
-                        )
-                    )
-                ).alias("m_standard"),
-                col(f"d.{dimension_name}") == col("m_standard.internal_code"),
-                "left",
-            )
-            # Join the data with the domain specific mapping for the specific dimension
-            .join(
-                F.broadcast(
-                    self.df_mapping_sdmx_codes.filter(
-                        (col("domain") == lit(self.domain_code))
-                        & (col("var_type") == lit(dimension_type))
-                        & (
-                            col("mapping_type").isNull()
-                            | (col("mapping_type") == lit(""))
-                        )
-                    )
-                ).alias("m_domain"),
-                col(f"d.{dimension_name}") == col("m_domain.internal_code"),
-                "left",
-            )
-            # Select only the columns we are interested in (this step is optional but recommended for debugging)
-            .select(
-                "d.*",
-                col("m_standard.external_code").alias("standard_external_code"),
-                col("m_standard.delete").alias("standard_delete"),
-                col("m_standard.multiplier").alias("standard_multiplier"),
-                col("m_domain.external_code").alias("domain_specific_external_code"),
-                col("m_domain.delete").alias("domain_specific_delete"),
-                col("m_domain.multiplier").alias("domain_specific_multiplier"),
-            )
-            # Filter out records to delete
-            .filter(
-                # Evaluate first the domain specific flag
-                F.when(
-                    col("domain_specific_delete").isNotNull(),
-                    ~col("domain_specific_delete"),
-                )
-                # Then evaluate the general flag
-                .when(
-                    col("standard_delete").isNotNull(), ~col("standard_delete")
-                ).otherwise(lit(True))
-            )
-            .withColumn(
-                dimension_name,
-                # Evaluate first the domain specific mapping
-                F.when(
-                    col("domain_specific_external_code").isNotNull(),
-                    col("domain_specific_external_code"),
-                )
-                # Then evaluate the general mapping
-                .when(
-                    col("standard_external_code").isNotNull(),
-                    col("standard_external_code"),
-                ).otherwise(col(dimension_name)),
-            )
-            .withColumn(
-                "value",
-                # Multiply first by the domain specific multiplier
-                F.when(
-                    col("domain_specific_multiplier").isNotNull(),
-                    col("value") * col("domain_specific_multiplier"),
-                )
-                # Then multiply by the general multiplier
-                .when(
-                    col("standard_external_code").isNotNull(),
-                    col("value") * col("standard_multiplier"),
-                ).otherwise(col("value")),
-            )
-            # Remove the columns that were not in the original dataset
-            .drop(
-                "standard_external_code",
-                "standard_delete",
-                "standard_multiplier",
-                "domain_specific_external_code",
-                "domain_specific_delete",
-                "domain_specific_multiplier",
-            )
-        )
-
-    def apply_sdmx_dimension_codes_mapping(self, df: DataFrame) -> DataFrame:
-        logging.info("Mapping codes to comply with SDMX standard")
-        for dimension_name, dimension_type in self.codelist_type_mapping.items():
-            df = df.transform(
-                self._apply_sdmx_dimension_codes_mapping_single,
-                dimension_name=dimension_name,
-                dimension_type=dimension_type,
-            )
-
-        return df
-
-    def drop_non_sdmx_columns(self, df: DataFrame) -> DataFrame:
-        cols_to_drop = [
-            row["internal_name"]
-            for row in self.df_mapping_sdmx_col_names.collect()
-            if row["delete"] is True
-        ]
-        logging.info(f"Dropping non-SDMX columns: {cols_to_drop}")
-        return df.drop(*cols_to_drop)
-
-    def apply_sdmx_column_names_mapping(self, df: DataFrame) -> DataFrame:
-        logging.info("Renaming columns to comply with SDMX standard")
-
-        mapping_sws_col_sdmx_col = {
-            row["internal_name"]: row["external_name"]
-            for row in self.df_mapping_sdmx_col_names.filter(
-                col("internal_name").isNotNull()
-                & (col("internal_name") != lit(""))
-                & ~col("delete")
-            ).collect()
-        }
-
-        logging.info(f"Column names mappings: {mapping_sws_col_sdmx_col}")
-
-        return df.withColumnsRenamed(mapping_sws_col_sdmx_col)
-
-    def add_sdmx_default_columns(self, df: DataFrame) -> DataFrame:
-        col_w_default_value = {
-            row["external_name"]: row["default_value"]
-            for row in self.df_mapping_sdmx_col_names.collect()
-            if row["add"] is True
-        }
-
-        logging.info("Adding SDMX columns with default values")
-
-        for name, default_value in col_w_default_value.items():
-            logging.info(
-                f"Adding SDMX column {name} with default value {default_value}"
-            )
-            df = df.withColumn(name, lit(default_value))
-
-        return df
-
-    def rearrange_sdmx_columns(self, df: DataFrame) -> DataFrame:
-        logging.info(
-            "Rearranging the columns to have the following order: Dimensions, TimeDimension, PrimaryMeasure, Attributes"
-        )
-
-        get_columns_for_type = lambda df, type: [
-            row[0]
-            for row in df.filter(col("type") == lit(type))
-            .select("external_name")
-            .collect()
-        ]
-
-        df_mapping_sdmx_no_del = self.df_mapping_sdmx_col_names.filter(~col("delete"))
-
-        dimensions = get_columns_for_type(df_mapping_sdmx_no_del, "Dimension")
-        time_dimensions = get_columns_for_type(df_mapping_sdmx_no_del, "TimeDimension")
-        primary_measure = get_columns_for_type(df_mapping_sdmx_no_del, "PrimaryMeasure")
-        attributes = get_columns_for_type(df_mapping_sdmx_no_del, "Attribute")
-
-        logging.info(f"Dimensions: {dimensions}")
-        logging.info(f"Time Dimensions: {time_dimensions}")
-        logging.info(f"Primary Measure: {primary_measure}")
-        logging.info(f"Attributes: {attributes}")
-
-        return df.select(*dimensions, *time_dimensions, *primary_measure, *attributes)
-
     def gen_gold_sws_disseminated_data(self) -> DataFrame:
         return (
             self.spark.read.option("tag", self.tag_name)
@@ -429,19 +104,6 @@ class SWSGoldIcebergSparkHelper:
             .transform(self.keep_dim_val_attr_columns)
         )
 
-    def gen_gold_sdmx_data(self) -> DataFrame:
-        return (
-            self.spark.read.option("tag", self.tag_name)
-            .table(self.iceberg_tables.SILVER.iceberg_id)
-            .transform(self.apply_diss_flag_filter)
-            .transform(self.apply_uom_mapping)
-            .transform(self.keep_dim_uom_val_attr_columns)
-            .transform(self.apply_sdmx_dimension_codes_mapping)
-            .transform(self.apply_sdmx_column_names_mapping)
-            .transform(self.add_sdmx_default_columns)
-            .transform(self.rearrange_sdmx_columns)
-        )
-
     def write_gold_sws_validated_data_to_iceberg_and_csv(
         self, df: DataFrame
     ) -> DataFrame:
@@ -550,14 +212,14 @@ class SWSGoldIcebergSparkHelper:
         df.writeTo(self.iceberg_tables.GOLD_PRE_SDMX.iceberg_id).createOrReplace()
 
         logging.info(
-            f"Gold SDMX table written to {self.iceberg_tables.GOLD_PRE_SDMX.iceberg_id}"
+            f"Gold pre-SDMX table written to {self.iceberg_tables.GOLD_PRE_SDMX.iceberg_id}"
         )
 
         self.spark.sql(
             f"ALTER TABLE {self.iceberg_tables.GOLD_PRE_SDMX.iceberg_id} CREATE OR REPLACE TAG `{self.tag_name}`"
         )
 
-        logging.info(f"gold PRE SDMX tag '{self.tag_name}' created")
+        logging.info(f"gold pre-SDMX tag '{self.tag_name}' created")
 
         df_1 = df.coalesce(1)
 
@@ -570,13 +232,6 @@ class SWSGoldIcebergSparkHelper:
 
         return df
 
-    def gen_and_write_gold_sdmx_data_to_iceberg_and_csv(self) -> DataFrame:
-        self.df_gold_sdmx = self.gen_gold_sdmx_data()
-
-        self.write_gold_sdmx_data_to_iceberg_and_csv(self.df_gold_sdmx)
-
-        return self.df_gold_sdmx
-
     def write_gold_sws_validated_sws_dissemination_tag(
         self, df: DataFrame, tags: Tags
     ) -> DataFrame:
@@ -733,7 +388,7 @@ class SWSGoldIcebergSparkHelper:
 
         new_iceberg_table = BaseDisseminatedTagTable(
             id=f"{self.domain_code.lower()}_gold_pre_sdmx_iceberg",
-            name=f"{self.domain_code} gold PRE SDMX Iceberg",
+            name=f"{self.domain_code} gold pre-SDMX Iceberg",
             description="Gold table containing all the cleaned data in SDMX compatible format, ready to be mapped using FMR",
             layer=TableLayer.GOLD,
             private=True,
@@ -754,7 +409,7 @@ class SWSGoldIcebergSparkHelper:
 
         new_diss_table = BaseDisseminatedTagTable(
             id=f"{self.domain_code.lower()}_gold_pre_sdmx_csv",
-            name=f"{self.domain_code} gold PRE SDMX csv",
+            name=f"{self.domain_code} gold pre-SDMX csv",
             description="Gold table containing all the cleaned data in SDMX compatible format, ready to be mapped using FMR and cached in csv",
             layer=TableLayer.GOLD,
             private=True,

{sws_spark_dissemination_helper-0.0.84 → sws_spark_dissemination_helper-0.0.85}/src/sws_spark_dissemination_helper/SWSPostgresSparkReader.py

@@ -299,6 +299,62 @@ class SWSPostgresSparkReader:
             unique_columns=["old_code"],
         )
 
+    def get_domain_code_source_datasets_ids_dest_dataset_id(
+        self, dataset_id: str, domain_code: str = None
+    ) -> Tuple[str, List[str], str]:
+        mapping_domains_id_df = self.read_pg_table(
+            pg_table=DatasetDatatables.MAPPING_DOMAINS_ID.id,
+            custom_schema=DatasetDatatables.MAPPING_DOMAINS_ID.schema,
+        )
+
+        if domain_code is None:
+            domain_code_df = mapping_domains_id_df.filter(
+                col("sws_source_id") == lit(dataset_id)
+            ).select("domain")
+
+            if domain_code_df.count() == 0:
+                raise ValueError(
+                    f'There is no row connecting the current source dataset id ({dataset_id}) to any domain in the table "{DatasetDatatables.MAPPING_DOMAINS_ID.name}"'
+                )
+
+            if domain_code_df.count() > 1:
+                raise ValueError(
+                    f'There is more than one domain referencing the current source dataset id ({dataset_id}) in the table "{DatasetDatatables.MAPPING_DOMAINS_ID.name}", please specify the domain code you want to process in the parameters'
+                )
+
+            domain_code = domain_code_df.collect()[0][0]
+
+        source_datasets_ids = [
+            row[0]
+            for row in (
+                mapping_domains_id_df.filter(col("domain") == lit(domain_code))
+                .select("sws_source_id")
+                .collect()
+            )
+        ]
+        dest_datasets_id_df = (
+            mapping_domains_id_df.filter(col("domain") == lit(domain_code))
+            .select("sws_source_id")
+            .distinct()
+        )
+
+        if dest_datasets_id_df.count() == 0:
+            raise ValueError(
+                f'There is no row connecting the current source dataset id and domain pair ({dataset_id}, {domain_code}) to any destination dataset id in the table "{DatasetDatatables.MAPPING_DOMAINS_ID.name}"'
+            )
+        if dest_datasets_id_df.count() > 1:
+            raise ValueError(
+                f'The source dataset id and domain pair ({dataset_id}, {domain_code}) must point only to one destination dataset in the table "{DatasetDatatables.MAPPING_DOMAINS_ID.name}"'
+            )
+
+        dest_datasets_id = dest_datasets_id_df.collect()[0][0]
+
+        logging.info(f"domain code: {domain_code}")
+        logging.info(f"source datasets ids: {source_datasets_ids}")
+        logging.info(f"dest datasets ids: {dest_datasets_id}")
+
+        return (domain_code, source_datasets_ids, dest_datasets_id)
+
     def get_dest_dataset_id(self, domain_code: str, dataset_id: str) -> Tuple[str, str]:
 
         df = self.read_pg_table(

sws_spark_dissemination_helper-0.0.84/requirements.txt

@@ -1,23 +0,0 @@
-"annotated-types==0.7.0",
-"boto3==1.36.18",
-"botocore==1.36.18",
-"certifi==2025.1.31",
-"charset-normalizer==3.4.1",
-"idna==3.10",
-"jmespath==1.0.1",
-"numpy==2.0.2",
-"pandas==2.2.3",
-"py4j==0.10.9.7",
-"pydantic==2.10.6",
-"pydantic_core==2.27.2",
-"pyspark==3.5.4",
-"python-dateutil==2.9.0.post0",
-"python-dotenv==0.19.2",
-"pytz==2025.1",
-"requests==2.32.3",
-"s3transfer==0.11.2",
-"six==1.17.0",
-"sws_api_client==1.4.3",
-"typing_extensions==4.12.2",
-"tzdata==2025.1",
-"urllib3==1.26.20"