sws-spark-dissemination-helper 0.0.141__py3-none-any.whl → 0.0.171__py3-none-any.whl

sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py

@@ -1,7 +1,7 @@
 import logging
 import time
 from copy import copy
-from typing import Dict, List, Tuple
+from typing import Dict, List, Tuple, Union
 
 import pyspark.sql.functions as F
 from pyspark.sql import DataFrame, SparkSession
@@ -26,6 +26,9 @@ class SWSBronzeIcebergSparkHelper:
         domain_code: str,
         dataset_details: dict = None,
         dataset_tables: DatasetTables = None,
+        keep_history: bool = False,
+        write_csv: bool = True,
+        source_tag: Union[str, None] = None,
     ) -> None:
         self.spark: SparkSession = spark
         self.dataset_details: dict = dataset_details
@@ -36,6 +39,9 @@ class SWSBronzeIcebergSparkHelper:
         self.dataset_tables: DatasetTables = dataset_tables
         self.iceberg_tables: IcebergTables = iceberg_tables
         self.domain_code = domain_code
+        self.keep_history: bool = keep_history
+        self.write_csv: bool = write_csv
+        self.source_tag: Union[str, None] = source_tag
 
         if dataset_details is not None:
             (
@@ -83,6 +89,7 @@ class SWSBronzeIcebergSparkHelper:
                 self.df_obs_coord,
                 self.df_metadata,
                 self.df_meta_elem,
+                self.df_tag_observation,
             ) = self.raw_data
 
             (
@@ -92,10 +99,11 @@ class SWSBronzeIcebergSparkHelper:
                 self.df_meta_elem_type,
                 self.df_language,
                 self.df_unit_of_measure,
+                self.df_dataset,
                 self.dfs_dimension,
             ) = self.raw_reference_data
 
-            self.df_user = self.raw_operational_data
+            (self.df_user, self.df_tag) = self.raw_operational_data
 
     def _get_dim_time_flag_columns(self) -> Tuple[List[str], List[str], str, List[str]]:
         """Extract the dimension columns with time, without time, the time column and the flag columns names."""
@@ -150,7 +158,7 @@ class SWSBronzeIcebergSparkHelper:
 
         return dfs_dimension
 
-    def _prepare_element_uom(self) -> DataFrame:
+    def _prepare_element_uom(self) -> Union[DataFrame, None]:
         """Prepare the element and unit of measure join."""
 
         # Get the element DataFrame
@@ -162,23 +170,24 @@ class SWSBronzeIcebergSparkHelper:
             if dimension_column == self.element_column
         )
 
-        # Join the element and the unit_of_measure
-        df_element_uom = (
-            df_element.alias("e")
-            .join(
-                self.df_unit_of_measure.alias("u"),
-                col("e.unit_of_measure") == col("u.id"),
-            )
-            .select(
-                col("e.code").alias("element_code"),
-                col("u.code").alias("unit_of_measure"),
-                col("u.symbol").alias("unit_of_measure_symbol"),
-                col("u.base_unit").alias("unit_of_measure_base_unit"),
-                col("u.multiplier").alias("unit_of_measure_multiplier"),
+        if any("unit_of_measure" == column.lower() for column in df_element.columns):
+            # Join the element and the unit_of_measure
+            df_element_uom = (
+                df_element.alias("e")
+                .join(
+                    self.df_unit_of_measure.alias("u"),
+                    col("e.unit_of_measure") == col("u.id"),
+                )
+                .select(
+                    col("e.code").alias("element_code"),
+                    col("u.code").alias("unit_of_measure"),
+                    col("u.symbol").alias("unit_of_measure_symbol"),
+                    col("u.base_unit").alias("unit_of_measure_base_unit"),
+                    col("u.multiplier").alias("unit_of_measure_multiplier"),
+                )
             )
-        )
 
-        return df_element_uom
+            return df_element_uom
 
     def _gen_denormalized_observation(self) -> DataFrame:
         """Original query upon which the below computation is based
@@ -270,20 +279,170 @@ class SWSBronzeIcebergSparkHelper:
                 .withColumnRenamed("code", dimension_column)
             )
 
-        df_intermediate = (
-            df_intermediate.alias("d")
-            .join(
-                F.broadcast(df_element_uom).alias("e"),
-                col(f"d.{self.element_column}") == col("e.element_code"),
-                "left",
+        if df_element_uom is not None:
+            df_intermediate = (
+                df_intermediate.alias("d")
+                .join(
+                    F.broadcast(df_element_uom).alias("e"),
+                    col(f"d.{self.element_column}") == col("e.element_code"),
+                    "left",
+                )
+                .drop("element_code")
             )
-            .drop("element_code")
-        )
 
         df_obs_denorm = df_intermediate
 
         return df_obs_denorm
 
+    def _gen_denormalized_observation_sql(self) -> DataFrame:
+        # ----------------
+        # Prepare dataframes for the joins
+        # ----------------
+
+        select_statement = """
+            o.id,
+            o.value,
+            u.email,
+            o.created_on,
+            o.replaced_on,
+            o.version"""
+
+        from_statement = f"""
+        FROM {self.dataset_tables.OBSERVATION.iceberg_id} o
+            JOIN {self.dataset_tables.USER.iceberg_id} u ON u.id = o.created_by
+            LEFT JOIN {self.dataset_tables.OBSERVATION_COORDINATE.iceberg_id} AS oc ON oc.id = o.observation_coordinates"""
+
+        hint_statement = ""
+
+        id_to_flag_col_mapping = {v: k for k, v in self.flag_col_to_id_mapping.items()}
+        for flag_col in self.flag_columns:
+            select_statement += f",\no.{id_to_flag_col_mapping[flag_col]} AS {flag_col}"
+
+        id_to_dim_col_mapping = {v: k for k, v in self.dim_col_to_id_mapping.items()}
+        for i, (dim_col, cl) in enumerate(
+            zip(self.dim_columns_w_time, self.dataset_tables.CODELISTS)
+        ):
+            select_statement += f",\nd{i}.code AS {dim_col}"
+            from_statement += f"\nLEFT JOIN {cl.iceberg_id} d{i} ON d{i}.id = oc.{id_to_dim_col_mapping[dim_col]}"
+            hint_statement = (
+                hint_statement + f", BROADCAST({cl.iceberg_id})"
+                if hint_statement
+                else f"BROADCAST({cl.iceberg_id})"
+            )
+
+        hint_statement = "/*+ " + hint_statement + " */"
+
+        final_query = "SELECT " + hint_statement + select_statement + from_statement
+        if not self.keep_history:
+            final_query += "\nWHERE o.replaced_on IS NULL"
+
+        logging.info("Final query for merging observation and observation_coordinares")
+        logging.info(final_query)
+
+        df_obs_denorm = self.spark.sql(final_query)
+
+        df_element_uom = self._prepare_element_uom()
+
+        dfs_dimension_w_validity = self._convert_dim_start_end_date_to_data()
+
+        # Join all the dimension codelists
+        for dimension_column, df_dimension in zip(
+            self.dim_columns_w_time, dfs_dimension_w_validity
+        ):
+            logging.debug(f"Joining dimension column: {dimension_column}")
+            logging.debug(f"df_obs_denorm columns: {df_obs_denorm.columns}")
+            logging.debug(
+                f"Is dimension {dimension_column} in the dataframe? {dimension_column in df_obs_denorm.columns}"
+            )
+            df_obs_denorm = (
+                df_obs_denorm.alias("o")
+                .join(
+                    F.broadcast(df_dimension.withColumnRenamed("id", "join_id")).alias(
+                        "d"
+                    ),
+                    col(f"{dimension_column}") == col("d.code"),
+                )
+                .drop("code", "join_id")
+            )
+            logging.debug(f"After join count: {df_obs_denorm.count()}")
+
+        if df_element_uom is not None:
+            df_obs_denorm = (
+                df_obs_denorm.alias("d")
+                .join(
+                    F.broadcast(df_element_uom).alias("e"),
+                    col(f"d.{self.element_column}") == col("e.element_code"),
+                    "left",
+                )
+                .drop("element_code")
+            )
+            logging.debug(f"After uom count: {df_obs_denorm.count()}")
+
+        return df_obs_denorm
+
+    def _gen_denormalized_observation_sql_from_tag(self) -> DataFrame:
+        # ----------------
+        # Prepare dataframes for the joins
+        # ----------------
+
+        select_statement = """
+            o.id,
+            o.value,
+            u.email,
+            o.created_on,
+            o.replaced_on,
+            o.version"""
+
+        from_statement = f"""
+        FROM {self.dataset_tables.OBSERVATION.iceberg_id} o
+            INNER JOIN {self.dataset_tables.TAG_OBSERVATION.iceberg_id} to ON o.id = to.observation
+            INNER JOIN {self.dataset_tables.TAG.iceberg_id} t ON to.tag = t.id
+            INNER JOIN {self.dataset_tables.DATASET.iceberg_id} d ON t.dataset = d.id
+            LEFT JOIN {self.dataset_tables.USER.iceberg_id} u ON u.id = o.created_by
+            LEFT JOIN {self.dataset_tables.OBSERVATION_COORDINATE.iceberg_id} AS oc ON oc.id = o.observation_coordinates"""
+
+        hint_statement = ""
+
+        id_to_flag_col_mapping = {v: k for k, v in self.flag_col_to_id_mapping.items()}
+        for flag_col in self.flag_columns:
+            select_statement += f",\no.{id_to_flag_col_mapping[flag_col]} AS {flag_col}"
+
+        id_to_dim_col_mapping = {v: k for k, v in self.dim_col_to_id_mapping.items()}
+        for i, (dim_col, cl) in enumerate(
+            zip(self.dim_columns_w_time, self.dataset_tables.CODELISTS)
+        ):
+            select_statement += f",\nd{i}.code AS {dim_col}"
+            from_statement += f"\nLEFT JOIN {cl.iceberg_id} d{i} ON d{i}.id = oc.{id_to_dim_col_mapping[dim_col]}"
+            hint_statement = (
+                hint_statement + f", BROADCAST({cl.iceberg_id})"
+                if hint_statement
+                else f"BROADCAST({cl.iceberg_id})"
+            )
+
+        hint_statement = "/*+ " + hint_statement + " */"
+
+        # TODO Add tag name as a parameter
+        where_statement = (
+            f"\nWHERE t.name = '{self.source_tag}' AND d.xml_name = '{self.dataset_id}'"
+        )
+
+        final_query = (
+            "SELECT "
+            + hint_statement
+            + select_statement
+            + from_statement
+            + where_statement
+        )
+        if not self.keep_history:
+            final_query += "\n AND o.replaced_on IS NULL"
+
+        logging.info("Final query for merging observation and observation_coordinares")
+        logging.info(final_query)
+
+        df_obs_denorm = self.spark.sql(final_query)
+
+        return df_obs_denorm
+
     def _gen_denormalized_metadata(self) -> DataFrame:
         """Original query upon which the below computation is based
 
@@ -347,6 +506,32 @@ class SWSBronzeIcebergSparkHelper:
 
         return df_meta_denorm
 
+    def _gen_denormalized_metadata_sql(self) -> DataFrame:
+        # ----------------
+        # Generate denormalized observation table
+        # ----------------
+
+        logging.info("meta_denorm start")
+
+        df_meta_denorm = self.spark.sql(
+            f"""
+        select m.observation as observation_id,
+            mt.code as type,
+            met.code as element_type,
+            l.country_code as language,
+            me.value
+        from {self.dataset_tables.METADATA_ELEMENT.iceberg_id} me
+            left join {self.dataset_tables.METADATA.iceberg_id} m on m.id = me.metadata
+            left join {self.dataset_tables.METADATA_ELEMENT_TYPE.iceberg_id} met on met.id = me.metadata_element_type
+            left join {self.dataset_tables.METADATA_TYPE.iceberg_id} mt on mt.id = m.metadata_type
+            left join {self.dataset_tables.LANGUAGE.iceberg_id} l on l.id = m.language
+        """
+        )
+
+        logging.info("meta_denorm write")
+
+        return df_meta_denorm
+
     def _gen_grouped_metadata(self) -> DataFrame:
         return (
             self._gen_denormalized_metadata()
@@ -367,6 +552,26 @@ class SWSBronzeIcebergSparkHelper:
             .agg(F.collect_list("metadata").alias("metadata"))
         )
 
+    def _gen_grouped_metadata_sql(self) -> DataFrame:
+        return (
+            self._gen_denormalized_metadata_sql()
+            .select(
+                col("observation_id"),
+                F.create_map(
+                    lit("type"),
+                    col("type"),
+                    lit("element_type"),
+                    col("element_type"),
+                    lit("language"),
+                    col("language"),
+                    lit("value"),
+                    col("value"),
+                ).alias("metadata"),
+            )
+            .groupby("observation_id")
+            .agg(F.collect_list("metadata").alias("metadata"))
+        )
+
     def _gen_bronze_data(self) -> DataFrame:
         return (
             self._gen_denormalized_observation()
@@ -379,9 +584,37 @@ class SWSBronzeIcebergSparkHelper:
             .drop("m.observation_id")
         )
 
+    def _gen_bronze_data_sql(self) -> DataFrame:
+        return (
+            self._gen_denormalized_observation_sql()
+            .alias("o")
+            .join(
+                self._gen_grouped_metadata_sql().alias("m"),
+                col("o.id") == col("m.observation_id"),
+                "left",
+            )
+            .drop("m.observation_id")
+        )
+
+    def _gen_bronze_data_sql_from_tag(self) -> DataFrame:
+        return (
+            self._gen_denormalized_observation_sql_from_tag()
+            .alias("o")
+            .join(
+                self._gen_grouped_metadata_sql().alias("m"),
+                col("o.id") == col("m.observation_id"),
+                "left",
+            )
+            .drop("m.observation_id")
+        )
+
     # TODO decouple data generation and data writing
-    def write_bronze_data_to_iceberg_and_csv(self) -> DataFrame:
-        self.df_bronze = self._gen_bronze_data()
+    def write_bronze_data_to_iceberg_and_csv(self, sql=True) -> DataFrame:
+
+        if sql:
+            self.df_bronze = self._gen_bronze_data_sql()
+        else:
+            self.df_bronze = self._gen_bronze_data()
 
         self.df_bronze.writeTo(self.iceberg_tables.BRONZE.iceberg_id).createOrReplace()
 
@@ -449,12 +682,15 @@ class SWSBronzeIcebergSparkHelper:
         logging.info("Bronze Dissemination tags successfully written")
 
     def write_bronze_disseminated_tag_data_to_iceberg_and_csv(
-        self, dimensions: Dict[str, List[str]]
+        self, dimensions: Dict[str, List[str]] = {}, from_tag=False
     ) -> DataFrame:
 
-        self.disseminated_tag_df = self.df_bronze
+        if from_tag:
+            self.disseminated_tag_df = self._gen_bronze_data_sql_from_tag()
+        else:
+            self.disseminated_tag_df = self.df_bronze
 
-        if isinstance(dimensions, dict):
+        if not from_tag and dimensions is not None and len(dimensions) != 0:
             for dimension_name, codes in dimensions.items():
                 logging.info(f"dimension_name: {dimension_name}")
                 logging.info(f"codes: {codes}")
@@ -533,3 +769,29 @@ class SWSBronzeIcebergSparkHelper:
         logging.debug(f"Tag with Added csv Table: {tag}")
 
         logging.info("Bronze Disseminated tag with selection successfully written")
+
+
+1
+frozenset({"8", "4", "2", "5", "9", "1", "7", "6", "0", "3"})
+1
+1
+2
+frozenset({"8", "4", "2", "5", "9", "1", "7", "6", "0", "3"})
+2
+1
+1
+frozenset({"8", "4", "2", "5", "9", "1", "7", "6", "0", "3"})
+1
+1
+2
+frozenset({"8", "4", "2", "5", "9", "1", "7", "6", "0", "3"})
+2
+1
+1
+frozenset({"8", "4", "2", "5", "9", "1", "7", "6", "0", "3"})
+1
+1
+1
+frozenset({"8", "4", "2", "5", "9", "1", "7", "6", "0", "3"})
+1
+1

sws_spark_dissemination_helper/SWSEasyIcebergSparkHelper.py

@@ -1,6 +1,6 @@
 import logging
 from copy import copy
-from typing import Dict, List, Tuple
+from typing import Dict, List, Tuple, Union
 
 import pyspark.sql.functions as F
 from pyspark.sql import DataFrame, SparkSession
@@ -26,6 +26,7 @@ class SWSEasyIcebergSparkHelper:
         dataset_tables: DatasetTables = None,
         keep_history: bool = False,
         write_csv: bool = True,
+        source_tag: Union[str, None] = None,
     ) -> None:
         self.spark: SparkSession = spark
         self.dataset_details: dict = dataset_details
@@ -37,6 +38,7 @@ class SWSEasyIcebergSparkHelper:
         self.iceberg_tables: IcebergTables = iceberg_tables
         self.keep_history: bool = keep_history
         self.write_csv: bool = write_csv
+        self.source_tag: Union[str, None] = source_tag
 
         if dataset_details is not None:
             (
@@ -69,8 +71,10 @@ class SWSEasyIcebergSparkHelper:
                 self.df_obs_coord,
                 self.df_metadata,
                 self.df_meta_elem,
+                self.df_tag_observation,
             ) = self.raw_data
 
+            logging.info(self.raw_reference_data)
             (
                 self.df_flag_method,
                 self.df_flag_obs_status,
@@ -78,10 +82,11 @@ class SWSEasyIcebergSparkHelper:
                 self.df_meta_elem_type,
                 self.df_language,
                 self.df_unit_of_measure,
+                self.df_dataset,
                 self.dfs_dimension,
             ) = self.raw_reference_data
 
-            self.df_user = self.raw_operational_data
+            (self.df_user, self.df_tag) = self.raw_operational_data
 
     def _get_dim_time_flag_columns(self) -> Tuple[List[str], List[str], str, List[str]]:
         """Extract the dimension columns with time, without time, the time column and the flag columns names."""
@@ -271,6 +276,75 @@ class SWSEasyIcebergSparkHelper:
         if not self.keep_history:
             final_query += "\nWHERE o.replaced_on IS NULL"
 
+        logging.info("Final query for merging observation and observation_coordinates")
+        logging.info(final_query)
+
+        df_obs_denorm = self.spark.sql(final_query)
+
+        df_obs_denorm.writeTo(
+            self.iceberg_tables.DENORMALIZED_OBSERVATION.iceberg_id
+        ).createOrReplace()
+
+        logging.info(f"{self.iceberg_tables.DENORMALIZED_OBSERVATION.table} write")
+
+        return df_obs_denorm
+
+    def _gen_denormalized_observation_sql_from_tag(self) -> DataFrame:
+        # ----------------
+        # Prepare dataframes for the joins
+        # ----------------
+
+        select_statement = """
+            o.id,
+            o.value,
+            u.email,
+            o.created_on,
+            o.replaced_on,
+            o.version"""
+
+        from_statement = f"""
+        FROM {self.dataset_tables.OBSERVATION.iceberg_id} o
+            INNER JOIN {self.dataset_tables.TAG_OBSERVATION.iceberg_id} to ON o.id = to.observation
+            INNER JOIN {self.dataset_tables.TAG.iceberg_id} t ON to.tag = t.id
+            INNER JOIN {self.dataset_tables.DATASET.iceberg_id} d ON t.dataset = d.id
+            LEFT JOIN {self.dataset_tables.USER.iceberg_id} u ON u.id = o.created_by
+            LEFT JOIN {self.dataset_tables.OBSERVATION_COORDINATE.iceberg_id} AS oc ON oc.id = o.observation_coordinates"""
+
+        hint_statement = ""
+
+        id_to_flag_col_mapping = {v: k for k, v in self.flag_col_to_id_mapping.items()}
+        for flag_col in self.flag_columns:
+            select_statement += f",\no.{id_to_flag_col_mapping[flag_col]} AS {flag_col}"
+
+        id_to_dim_col_mapping = {v: k for k, v in self.dim_col_to_id_mapping.items()}
+        for i, (dim_col, cl) in enumerate(
+            zip(self.dim_columns_w_time, self.dataset_tables.CODELISTS)
+        ):
+            select_statement += f",\nd{i}.code AS {dim_col}"
+            from_statement += f"\nLEFT JOIN {cl.iceberg_id} d{i} ON d{i}.id = oc.{id_to_dim_col_mapping[dim_col]}"
+            hint_statement = (
+                hint_statement + f", BROADCAST({cl.iceberg_id})"
+                if hint_statement
+                else f"BROADCAST({cl.iceberg_id})"
+            )
+
+        hint_statement = "/*+ " + hint_statement + " */"
+
+        # TODO Add tag name as a parameter
+        where_statement = (
+            f"\nWHERE t.name = '{self.source_tag}' AND d.xml_name = '{self.dataset_id}'"
+        )
+
+        final_query = (
+            "SELECT "
+            + hint_statement
+            + select_statement
+            + from_statement
+            + where_statement
+        )
+        if not self.keep_history:
+            final_query += "\n AND o.replaced_on IS NULL"
+
         logging.info("Final query for merging observation and observation_coordinares")
         logging.info(final_query)
 
@@ -350,7 +424,13 @@ class SWSEasyIcebergSparkHelper:
 
         df_meta_denorm = self.spark.sql(
             f"""
-        select m.observation as observation_id,
+        select 
+            /*+
+                BROADCAST({self.dataset_tables.METADATA_ELEMENT_TYPE.iceberg_id}),
+                BROADCAST({self.dataset_tables.METADATA_TYPE.iceberg_id}),
+                BROADCAST({self.dataset_tables.LANGUAGE.iceberg_id})
+            */
+            m.observation as observation_id,
             mt.code as type,
             met.code as element_type,
             l.country_code as language,
@@ -363,7 +443,11 @@ class SWSEasyIcebergSparkHelper:
         """
         )
 
-        logging.info("meta_denorm write")
+        df_meta_denorm.writeTo(
+            self.iceberg_tables.DENORMALIZED_METADATA.iceberg_id
+        ).createOrReplace()
+
+        logging.info(f"{self.iceberg_tables.DENORMALIZED_METADATA.table} write")
 
         return df_meta_denorm
 
@@ -388,25 +472,31 @@ class SWSEasyIcebergSparkHelper:
         )
 
     def _gen_grouped_metadata_sql(self) -> DataFrame:
-        return (
-            self._gen_denormalized_metadata_sql()
-            .select(
-                col("observation_id"),
-                F.create_map(
-                    lit("type"),
-                    col("type"),
-                    lit("element_type"),
-                    col("element_type"),
-                    lit("language"),
-                    col("language"),
-                    lit("value"),
-                    col("value"),
-                ).alias("metadata"),
-            )
-            .groupby("observation_id")
-            .agg(F.collect_list("metadata").alias("metadata"))
+        df_meta_grouped = self.spark.sql(
+            f"""
+        SELECT
+            observation_id,
+            collect_list(
+                map(
+                    'type', type,
+                    'element_type', element_type,
+                    'language', language,
+                    'value', value
+                )
+            ) AS metadata
+        FROM {self.iceberg_tables.DENORMALIZED_METADATA.iceberg_id}
+        GROUP BY observation_id
+        """
         )
 
+        df_meta_grouped.writeTo(
+            self.iceberg_tables.GROUPED_METADATA.iceberg_id
+        ).createOrReplace()
+
+        logging.info(f"{self.iceberg_tables.GROUPED_METADATA.table} write")
+
+        return df_meta_grouped
+
     def _gen_denormalied_data(self) -> DataFrame:
         return (
             self._gen_denormalized_observation()
@@ -420,8 +510,24 @@ class SWSEasyIcebergSparkHelper:
         )
 
     def _gen_denormalied_data_sql(self) -> DataFrame:
+        self._gen_denormalized_observation_sql()
+        self._gen_denormalized_metadata_sql()
+        self._gen_grouped_metadata_sql()
+
+        return self.spark.sql(
+            f"""
+            SELECT
+                o.*,
+                m.metadata
+            FROM {self.iceberg_tables.DENORMALIZED_OBSERVATION.iceberg_id} AS o
+            LEFT JOIN {self.iceberg_tables.GROUPED_METADATA.iceberg_id} AS m
+                ON o.id = m.observation_id
+            """
+        )
+
+    def _gen_denormalied_data_sql_from_tag(self) -> DataFrame:
         return (
-            self._gen_denormalized_observation_sql()
+            self._gen_denormalized_observation_sql_from_tag()
             .alias("o")
             .join(
                 self._gen_grouped_metadata_sql().alias("m"),
@@ -431,7 +537,7 @@ class SWSEasyIcebergSparkHelper:
             .drop("m.observation_id")
         )
 
-    def write_data_to_iceberg_and_csv(self, sql=False) -> DataFrame:
+    def write_data_to_iceberg_and_csv(self, sql=True) -> DataFrame:
         if sql:
             self.df_denorm = self._gen_denormalied_data_sql()
         else:
@@ -504,18 +610,21 @@ class SWSEasyIcebergSparkHelper:
         logging.info("Unfiltered data tags successfully written")
 
     def write_filtered_data_to_iceberg_and_csv(
-        self, dimensions: Dict[str, List[str]]
+        self, dimensions: Dict[str, List[str]] = None, from_tag=False
     ) -> DataFrame:
 
-        self.filtered_df = self.df_denorm
+        if from_tag:
+            self.filtered_df = self._gen_denormalied_data_sql_from_tag()
+        else:
+            self.filtered_df = self.df_denorm
 
-        for dimension_name, codes in dimensions.items():
-            logging.info(f"dimension_name: {dimension_name}")
-            logging.info(f"codes: {codes}")
-            if len(codes) != 0:
-                self.filtered_df = self.filtered_df.filter(
-                    col(dimension_name).isin(codes)
-                )
+            for dimension_name, codes in dimensions.items():
+                logging.info(f"dimension_name: {dimension_name}")
+                logging.info(f"codes: {codes}")
+                if len(codes) != 0:
+                    self.filtered_df = self.filtered_df.filter(
+                        col(dimension_name).isin(codes)
+                    )
 
         self.filtered_df.writeTo(
             self.iceberg_tables.TABLE_FILTERED.iceberg_id
@@ -586,3 +695,29 @@ class SWSEasyIcebergSparkHelper:
             logging.debug(f"Tag with Added csv Table: {tag}")
 
         logging.info("Filtered data tags successfully written")
+
+
+1
+frozenset({"1", "0", "7", "9", "4", "8", "6", "3", "2", "5"})
+1
+1
+2
+frozenset({"1", "0", "7", "9", "4", "8", "6", "3", "2", "5"})
+2
+1
+1
+frozenset({"1", "0", "7", "9", "4", "8", "6", "3", "2", "5"})
+1
+1
+2
+frozenset({"1", "0", "7", "9", "4", "8", "6", "3", "2", "5"})
+2
+1
+1
+frozenset({"1", "0", "7", "9", "4", "8", "6", "3", "2", "5"})
+1
+1
+1
+frozenset({"1", "0", "7", "9", "4", "8", "6", "3", "2", "5"})
+1
+1

sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py

@@ -271,6 +271,60 @@ class SWSGoldIcebergSparkHelper:
 
         return df
 
+    def write_gold_faostat_data_to_iceberg_and_csv(self, df: DataFrame) -> DataFrame:
+        """The expected input to this function is the output of the sws disseminated function"""
+        df.writeTo(self.iceberg_tables.GOLD_FAOSTAT.iceberg_id).createOrReplace()
+
+        logging.info(
+            f"Gold FAOSTAT table written to {self.iceberg_tables.GOLD_FAOSTAT.iceberg_id}"
+        )
+
+        self.spark.sql(
+            f"ALTER TABLE {self.iceberg_tables.GOLD_FAOSTAT.iceberg_id} CREATE OR REPLACE TAG `{self.tag_name}`"
+        )
+
+        logging.info(f"gold FAOSTAT tag '{self.tag_name}' created")
+
+        df_1 = df.coalesce(1)
+
+        save_cache_csv(
+            df=df_1,
+            bucket=self.bucket,
+            prefix=self.iceberg_tables.GOLD_FAOSTAT.csv_prefix,
+            tag_name=self.tag_name,
+        )
+
+        return df
+
+    def write_gold_faostat_unfiltered_data_to_iceberg_and_csv(
+        self, df: DataFrame
+    ) -> DataFrame:
+        """The expected input to this function is the output of the sws disseminated function"""
+        df.writeTo(
+            self.iceberg_tables.GOLD_FAOSTAT_UNFILTERED.iceberg_id
+        ).createOrReplace()
+
+        logging.info(
+            f"Gold FAOSTAT unfiltered table written to {self.iceberg_tables.GOLD_FAOSTAT.iceberg_id}"
+        )
+
+        self.spark.sql(
+            f"ALTER TABLE {self.iceberg_tables.GOLD_FAOSTAT_UNFILTERED.iceberg_id} CREATE OR REPLACE TAG `{self.tag_name}`"
+        )
+
+        logging.info(f"gold FAOSTAT unfiltered tag '{self.tag_name}' created")
+
+        df_1 = df.coalesce(1)
+
+        save_cache_csv(
+            df=df_1,
+            bucket=self.bucket,
+            prefix=self.iceberg_tables.GOLD_FAOSTAT_UNFILTERED.csv_prefix,
+            tag_name=self.tag_name,
+        )
+
+        return df
+
     def write_gold_sws_validated_sws_dissemination_tag(
         self, df: DataFrame, tags: Tags
     ) -> DataFrame:
@@ -496,8 +550,8 @@ class SWSGoldIcebergSparkHelper:
         logging.debug(f"Tag with Added Iceberg Table: {tag}")
 
         new_diss_table = BaseDisseminatedTagTable(
-            id=f"{self.domain_code.lower()}_gold_pre_sdmx_csv",
-            name=f"{self.domain_code} gold pre-SDMX csv",
+            id=f"{self.domain_code.lower()}_gold_sws_csv",
+            name=f"{self.domain_code} gold SWS csv",
             description="Gold table containing the tag data without any processing cached in csv",
             layer=TableLayer.GOLD,
             private=True,
@@ -515,3 +569,101 @@ class SWSGoldIcebergSparkHelper:
         logging.debug(f"Tag with Added csv Table: {tag}")
 
         return df
+
+    def write_gold_faostat_dissemination_tag(
+        self, df: DataFrame, tags: Tags
+    ) -> DataFrame:
+        # Get or create a new tag
+        tag = get_or_create_tag(tags, self.dataset_id, self.tag_name, self.tag_name)
+        logging.debug(f"Tag: {tag}")
+
+        new_iceberg_table = BaseDisseminatedTagTable(
+            id=f"{self.domain_code.lower()}_gold_faostat_iceberg",
+            name=f"{self.domain_code} gold FAOSTAT Iceberg",
+            description="Gold table containing the tag data in FAOSTAT format",
+            layer=TableLayer.GOLD,
+            private=True,
+            type=TableType.ICEBERG,
+            database=IcebergDatabases.GOLD_DATABASE,
+            table=self.iceberg_tables.GOLD_FAOSTAT.table,
+            path=self.iceberg_tables.GOLD_FAOSTAT.path,
+            structure={"columns": df.schema.jsonValue()["fields"]},
+        )
+        tag = upsert_disseminated_table(
+            sws_tags=tags,
+            tag=tag,
+            dataset_id=self.dataset_id,
+            tag_name=self.tag_name,
+            table=new_iceberg_table,
+        )
+        logging.debug(f"Tag with Added Iceberg Table: {tag}")
+
+        new_diss_table = BaseDisseminatedTagTable(
+            id=f"{self.domain_code.lower()}_gold_faostat_csv",
+            name=f"{self.domain_code} gold FAOSTAT csv",
+            description="Gold table containing the tag data in FAOSTAT format in csv",
+            layer=TableLayer.GOLD,
+            private=True,
+            type=TableType.CSV,
+            path=self.iceberg_tables.GOLD_FAOSTAT.csv_path,
+            structure={"columns": df.schema.jsonValue()["fields"]},
+        )
+        tag = upsert_disseminated_table(
+            sws_tags=tags,
+            tag=tag,
+            dataset_id=self.dataset_id,
+            tag_name=self.tag_name,
+            table=new_diss_table,
+        )
+        logging.debug(f"Tag with Added csv Table: {tag}")
+
+        return df
+
+    def write_gold_faostat_unfiltered_dissemination_tag(
+        self, df: DataFrame, tags: Tags
+    ) -> DataFrame:
+        # Get or create a new tag
+        tag = get_or_create_tag(tags, self.dataset_id, self.tag_name, self.tag_name)
+        logging.debug(f"Tag: {tag}")
+
+        new_iceberg_table = BaseDisseminatedTagTable(
+            id=f"{self.domain_code.lower()}_gold_faostat_unfiltered_iceberg",
+            name=f"{self.domain_code} gold FAOSTAT unfiltered Iceberg",
+            description="Gold table containing all the tag data in FAOSTAT format",
+            layer=TableLayer.GOLD,
+            private=True,
+            type=TableType.ICEBERG,
+            database=IcebergDatabases.GOLD_DATABASE,
+            table=self.iceberg_tables.GOLD_FAOSTAT_UNFILTERED.table,
+            path=self.iceberg_tables.GOLD_FAOSTAT_UNFILTERED.path,
+            structure={"columns": df.schema.jsonValue()["fields"]},
+        )
+        tag = upsert_disseminated_table(
+            sws_tags=tags,
+            tag=tag,
+            dataset_id=self.dataset_id,
+            tag_name=self.tag_name,
+            table=new_iceberg_table,
+        )
+        logging.debug(f"Tag with Added Iceberg Table: {tag}")
+
+        new_diss_table = BaseDisseminatedTagTable(
+            id=f"{self.domain_code.lower()}_gold_faostat_unfiltered_csv",
+            name=f"{self.domain_code} gold FAOSTAT unfiltered csv",
+            description="Gold table containing the tag data in FAOSTAT format in csv",
+            layer=TableLayer.GOLD,
+            private=True,
+            type=TableType.CSV,
+            path=self.iceberg_tables.GOLD_FAOSTAT_UNFILTERED.csv_path,
+            structure={"columns": df.schema.jsonValue()["fields"]},
+        )
+        tag = upsert_disseminated_table(
+            sws_tags=tags,
+            tag=tag,
+            dataset_id=self.dataset_id,
+            tag_name=self.tag_name,
+            table=new_diss_table,
+        )
+        logging.debug(f"Tag with Added csv Table: {tag}")
+
+        return df

sws_spark_dissemination_helper/SWSPostgresSparkReader.py

@@ -94,25 +94,37 @@ class SWSPostgresSparkReader:
 
             logging.info(f"{pg_table} read start")
 
-            # Read observations from the PostgreSQL table into a DataFrame
-            df = (
-                self.spark.read.format("jdbc")
-                .option("customSchema", custom_schema)
-                .option("dbtable", pg_table)
-                .option("partitionColumn", partition_column)
-                .option("lowerBound", min_id)
-                .option("upperBound", max_id)
-                .option("numPartitions", num_partitions)
-                .option("fetchsize", "1000")
-                .option("url", self.jdbc_url)
-                .option("user", self.jdbc_conn_properties["user"])
-                .option("password", self.jdbc_conn_properties["password"])
-                .option("driver", SPARK_POSTGRES_DRIVER)
-                .load()
-                # .repartition(1024, partition_column)
-                # .sortWithinPartitions(partition_column)
-                # .cache()
-            )
+            if min_id is None or max_id is None:
+                df = (
+                    self.spark.read.format("jdbc")
+                    .option("customSchema", custom_schema)
+                    .option("dbtable", pg_table)
+                    .option("fetchsize", "1000")
+                    .option("url", self.jdbc_url)
+                    .option("user", self.jdbc_conn_properties["user"])
+                    .option("password", self.jdbc_conn_properties["password"])
+                    .option("driver", SPARK_POSTGRES_DRIVER)
+                    .load()
+                )
+            else:
+                df = (
+                    self.spark.read.format("jdbc")
+                    .option("customSchema", custom_schema)
+                    .option("dbtable", pg_table)
+                    .option("partitionColumn", partition_column)
+                    .option("lowerBound", min_id)
+                    .option("upperBound", max_id)
+                    .option("numPartitions", num_partitions)
+                    .option("fetchsize", "1000")
+                    .option("url", self.jdbc_url)
+                    .option("user", self.jdbc_conn_properties["user"])
+                    .option("password", self.jdbc_conn_properties["password"])
+                    .option("driver", SPARK_POSTGRES_DRIVER)
+                    .load()
+                    # .repartition(1024, partition_column)
+                    # .sortWithinPartitions(partition_column)
+                    # .cache()
+                )
         else:
             df = (
                 self.spark.read.format("jdbc")
@@ -195,6 +207,7 @@ class SWSPostgresSparkReader:
             (dataset_tables.OBSERVATION_COORDINATE, "id", 10),
             (dataset_tables.METADATA, "id", 10),
             (dataset_tables.METADATA_ELEMENT, "metadata", 10),
+            (dataset_tables.TAG_OBSERVATION, "tag", 10),
         ]
         return self._import_tables(data_tables)
 
@@ -209,25 +222,30 @@ class SWSPostgresSparkReader:
             dataset_tables.METADATA_ELEMENT_TYPE,
             dataset_tables.LANGUAGE,
             dataset_tables.UNIT_OF_MEASURE,
+            dataset_tables.DATASET,
             *dataset_tables.CODELISTS,
         ]
+        logging.info(
+            f"Importing reference data tables: {[(table.postgres_id, table.iceberg_id) for table in reference_data_tables]}"
+        )
         return self._import_tables(
             [(table, None, 1) for table in reference_data_tables]
         )
 
     def import_operational_data_tables(
         self, dataset_tables: DatasetTables
-    ) -> DataFrame:
+    ) -> List[DataFrame]:
         # Define and import operational data table without partitioning
         operational_data_tables = [
             (dataset_tables.USER, None, 1),
+            (dataset_tables.TAG, None, 1),
         ]
-        return self._import_tables(operational_data_tables)[0]
+        return self._import_tables(operational_data_tables)
 
     def import_data_reference_data_operational_data(
         self, dataset_tables: DatasetTables
     ) -> Tuple[
-        Tuple[DataFrame, DataFrame, DataFrame, DataFrame],
+        Tuple[DataFrame, DataFrame, DataFrame, DataFrame, DataFrame],
         Tuple[
             DataFrame,
             DataFrame,
@@ -235,22 +253,23 @@ class SWSPostgresSparkReader:
             DataFrame,
             DataFrame,
             DataFrame,
+            DataFrame,
             List[DataFrame],
         ],
-        DataFrame,
+        Tuple[DataFrame, DataFrame],
     ]:
         # Import and organize DataFrames into the desired output structure
         data_dfs = self.import_data_tables(dataset_tables)
         reference_data_dfs = self.import_reference_data_tables(dataset_tables)
-        operational_data_df = self.import_operational_data_tables(dataset_tables)
+        operational_data_dfs = self.import_operational_data_tables(dataset_tables)
 
         return (
             tuple(data_dfs),
             (
-                *reference_data_dfs[:6],
-                reference_data_dfs[6:],
+                *reference_data_dfs[:7],
+                reference_data_dfs[7:],
             ),
-            operational_data_df,
+            tuple(operational_data_dfs),
         )
 
     def get_codelist_type_mapping(

sws_spark_dissemination_helper/constants.py

@@ -1,3 +1,5 @@
+from typing import List
+
 from pyspark.sql.functions import col, lit
 
 SPARK_POSTGRES_DRIVER = "org.postgresql.Driver"
@@ -34,10 +36,14 @@ class DomainFilters:
 class DatasetDatatables:
 
     class __SWSDatatable:
-        def __init__(self, id: str, name: str, schema: str):
+        def __init__(
+            self, id: str, name: str, schema: str, join_columns: List[str] = []
+        ):
             self.id = id
+            self.iceberg_id = f"{IcebergDatabases.BRONZE_DATABASE}.{id.split('.')[1]}"
             self.name = name
             self.schema = schema
+            self.join_columns = join_columns
 
     # Aggregation Tables
     AGGREGATES_COMPOSITION = __SWSDatatable(
@@ -50,22 +56,37 @@ class DatasetDatatables:
         name="Aggregation - Aggregates per elements",
         schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, element STRING, aggregation_type STRING, code STRING",
     )
-    
+
     # Dissemination Tables
     DISSEMINATION_TYPE_LIST = __SWSDatatable(
         id="datatables.dissemination_{type}_list",
         name="Dissemination - {type} list",
         schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, code STRING, name STRING, aggregation_type STRING, dissemination BOOLEAN, aggregation BOOLEAN",
+        join_columns=["domain", "code"],
     )
     DISSEMINATION_EXCEPTIONS = __SWSDatatable(
         id="datatables.dissemination_exception",
         name="Dissemination - Exceptions",
         schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, dim1_code STRING, dim2_code STRING, dim3_code STRING, dim4_code STRING, dim5_code STRING, dim6_code STRING, dim7_code STRING, status_flag STRING, method_flag STRING, dissemination BOOLEAN, aggregation BOOLEAN, note STRING",
+        join_columns=[
+            "domain",
+            " dim1_code",
+            " dim2_code",
+            " dim3_code",
+            " dim4_code",
+            " dim5_code",
+            " dim6_code",
+            " dim7_code",
+            " status_flag",
+            " method_flag",
+        ],
     )
+    # TODO Deprecate
     DISSEMINATION_ITEM_LIST_FAOSTAT = __SWSDatatable(
         id="datatables.dissemination_item_list_faostat",
         name="Dissemination - Item list - FAOSTAT",
         schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, code STRING, name STRING, aggregation_type STRING, dissemination BOOLEAN, aggregation BOOLEAN",
+        join_columns=["domain", "code"],
     )
 
     # Mapping Tables
@@ -73,19 +94,23 @@ class DatasetDatatables:
         id="datatables.aggregates_mapping_domains_id",
         name="Mapping - Domains ID",
         schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, domain_name STRING, sws_source_id STRING, sws_destination_id STRING",
+        join_columns=["domain", "sws_source_id"],
     )
     MAPPING_CODELIST_TYPE = __SWSDatatable(
         id="datatables.mapping_codelist_type",
         name="Mapping Codelist type",
         schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, col_name STRING, col_type STRING",
+        join_columns=["domain", "col_name"],
     )
     MAPPING_CODE_CORRECTION = __SWSDatatable(
         id="datatables.aggregates_mapping_code_correction",
         name="Mapping - Code correction",
         schema=f"{DATATABLE_COLUMNS_SCHEMA}, domain STRING, old_code STRING, new_code STRING, var_type STRING, delete BOOLEAN, multiplier FLOAT, mapping_type STRING",
+        join_columns=["domain", "old_code", "var_type", "mapping_type"],
     )
 
     # Non-SWS Sources Tables
+    # TODO To deprecate
     FAOSTAT_CODE_MAPPING = __SWSDatatable(
         id="datatables.faostat_code_mapping",
         name="FAOSTAT Code Mapping",
@@ -147,6 +172,11 @@ class DatasetTables:
             iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.{self.__dataset_id}_metadata_element",
             schema="id BIGINT, metadata INT, metadata_element_type INT, value STRING",
         )
+        self.TAG_OBSERVATION = self.__SWSTable(
+            postgres_id=f"{self.__dataset_id}.tag_observation",
+            iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.{self.__dataset_id}_tag_observation",
+            schema="tag BIGINT, observation INT",
+        )
 
         # Reference data
         self.CODELISTS = [
@@ -178,18 +208,21 @@ class DatasetTables:
         iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.metadata_element_type",
         schema="id INT, metadata_type INT, code STRING, description STRING, mandatory BOOLEAN, repeatable BOOLEAN, private BOOLEAN",
     )
-
     LANGUAGE = __SWSTable(
         postgres_id="reference_data.language",
         iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.language",
         schema="id INT, country_code STRING, description STRING",
     )
-
     UNIT_OF_MEASURE = __SWSTable(
         postgres_id="reference_data.unit_of_measure",
         iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.unit_of_measure",
         schema="id INT, code STRING, sdmx_code STRING, metric BOOLEAN, description STRING, symbol STRING, base_unit STRING, multiplier DECIMAL",
     )
+    DATASET = __SWSTable(
+        postgres_id="reference_data.dataset",
+        iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.dataset",
+        schema="id INT, xml_name STRING",
+    )
 
     # Operational data
     USER = __SWSTable(
@@ -197,6 +230,11 @@ class DatasetTables:
         iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.user",
         schema="id INT, username STRING, preferences INT, email STRING, active BOOLEAN, settings STRING",
     )
+    TAG = __SWSTable(
+        postgres_id="operational_data.tag",
+        iceberg_id=f"{IcebergDatabases.STAGING_DATABASE}.tag",
+        schema="id INT, name STRING, reference_date DATE, dataset INT, type STRING, released_ON DATE, released_by INT, properties STRING",
+    )
 
 
 class IcebergTable:
@@ -216,30 +254,37 @@ class IcebergTables:
         self.__tag_name = tag_name
 
         # TODO Fix later with a more appropriate DATABASE
-        self.TABLE = self._create_iceberg_table("BRONZE")
-        self.TABLE_FILTERED = self._create_iceberg_table("BRONZE", suffix="filtered")
-        self.BRONZE = self._create_iceberg_table("BRONZE")
-        self.BRONZE_DISS_TAG = self._create_iceberg_table("BRONZE", suffix="diss_tag")
-        self.SILVER = self._create_iceberg_table("SILVER", prefix=domain)
+        self.DENORMALIZED_OBSERVATION = self.create_iceberg_table("BRONZE", suffix="denormalized_observation")
+        self.DENORMALIZED_METADATA = self.create_iceberg_table("BRONZE", suffix="denormalized_metadata")
+        self.GROUPED_METADATA = self.create_iceberg_table("BRONZE", suffix="grouped_metadata")
+        self.TABLE = self.create_iceberg_table("BRONZE")
+        self.TABLE_FILTERED = self.create_iceberg_table("BRONZE", suffix="filtered")
+        self.BRONZE = self.create_iceberg_table("BRONZE")
+        self.BRONZE_DISS_TAG = self.create_iceberg_table("BRONZE", suffix="diss_tag")
+        self.SILVER = self.create_iceberg_table("SILVER", prefix=domain)
 
         # GOLD tables with specific suffixes
-        self.GOLD_SWS = self._create_iceberg_table(
-            "GOLD", prefix=domain, suffix="sws"
-        )
-        self.GOLD_SDMX = self._create_iceberg_table(
+        self.GOLD_SWS = self.create_iceberg_table("GOLD", prefix=domain, suffix="sws")
+        self.GOLD_SDMX = self.create_iceberg_table(
             "GOLD", prefix=domain, suffix="sdmx_disseminated"
         )
-        self.GOLD_SWS_VALIDATED = self._create_iceberg_table(
+        self.GOLD_SWS_VALIDATED = self.create_iceberg_table(
             "GOLD", prefix=domain, suffix="sws_validated"
         )
-        self.GOLD_SWS_DISSEMINATED = self._create_iceberg_table(
+        self.GOLD_SWS_DISSEMINATED = self.create_iceberg_table(
             "GOLD", prefix=domain, suffix="sws_disseminated"
         )
-        self.GOLD_PRE_SDMX = self._create_iceberg_table(
+        self.GOLD_PRE_SDMX = self.create_iceberg_table(
             "GOLD", prefix=domain, suffix="pre_sdmx"
         )
+        self.GOLD_FAOSTAT = self.create_iceberg_table(
+            "GOLD", prefix=domain, suffix="faostat"
+        )
+        self.GOLD_FAOSTAT_UNFILTERED = self.create_iceberg_table(
+            "GOLD", prefix=domain, suffix="faostat_unfiltered"
+        )
 
-    def _create_iceberg_table(
+    def create_iceberg_table(
         self, level: str, prefix: str = "", suffix: str = ""
     ) -> IcebergTable:
         database = getattr(IcebergDatabases, f"{level}_DATABASE")

{sws_spark_dissemination_helper-0.0.141.dist-info → sws_spark_dissemination_helper-0.0.171.dist-info}/METADATA

@@ -1,8 +1,8 @@
 Metadata-Version: 2.4
 Name: sws-spark-dissemination-helper
-Version: 0.0.141
+Version: 0.0.171
 Summary: A Python helper package providing streamlined Spark functions for efficient data dissemination processes
-Project-URL: Repository, https://bitbucket.org/cioapps/sws-it-python-spark-dissemination-helper
+Project-URL: Repository, https://github.com/un-fao/fao-sws-it-python-spark-dissemination-helper
 Author-email: Daniele Mansillo <danielemansillo@gmail.com>
 License: MIT License
         
@@ -31,27 +31,27 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.9
 Requires-Dist: annotated-types==0.7.0
-Requires-Dist: boto3==1.36.18
-Requires-Dist: botocore==1.36.18
+Requires-Dist: boto3>=1.40.75
+Requires-Dist: botocore>=1.40.75
 Requires-Dist: certifi==2025.1.31
 Requires-Dist: charset-normalizer==3.4.1
-Requires-Dist: idna==3.10
+Requires-Dist: idna>=3.10
 Requires-Dist: jmespath==1.0.1
 Requires-Dist: numpy==2.0.2
-Requires-Dist: pandas==2.2.3
+Requires-Dist: pandas==2.3.3
 Requires-Dist: py4j==0.10.9.7
 Requires-Dist: pydantic-core==2.27.2
 Requires-Dist: pydantic==2.10.6
 Requires-Dist: pyspark==3.5.4
 Requires-Dist: python-dateutil==2.9.0.post0
 Requires-Dist: python-dotenv==0.19.2
-Requires-Dist: pytz==2025.1
+Requires-Dist: pytz==2025.2
 Requires-Dist: requests==2.32.3
-Requires-Dist: s3transfer==0.11.2
+Requires-Dist: s3transfer>=0.11.2
 Requires-Dist: six==1.17.0
-Requires-Dist: sws-api-client>=1.5.1
-Requires-Dist: typing-extensions==4.12.2
-Requires-Dist: tzdata==2025.1
+Requires-Dist: sws-api-client==2.3.0
+Requires-Dist: typing-extensions>=4.12.2
+Requires-Dist: tzdata==2025.2
 Requires-Dist: urllib3==1.26.20
 Description-Content-Type: text/markdown
 

{sws_spark_dissemination_helper-0.0.141.dist-info → sws_spark_dissemination_helper-0.0.171.dist-info}/RECORD

@@ -1,13 +1,13 @@
-sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py,sha256=rHTPkMU8_DqP-MrHz7CzeVmrfyU2s7eJfk9gIDyTwio,20076
+sws_spark_dissemination_helper/SWSBronzeIcebergSparkHelper.py,sha256=N0eQ2LXtpPeZQCWYi85sMLmpXRzLA2erECiba8tqOAY,29595
 sws_spark_dissemination_helper/SWSDatatablesExportHelper.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sws_spark_dissemination_helper/SWSEasyIcebergSparkHelper.py,sha256=6Sgn2anJedjrzwUA6kfzO37QyrV2Q0ufcvgFE--DtVo,21823
-sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py,sha256=KZy6v4V3ugzKq_0L8JLmTPClN0hx-9uWpAwNFcs37Og,19339
-sws_spark_dissemination_helper/SWSPostgresSparkReader.py,sha256=cRLa6KYv7tbGB2_Rpcyvb3ksGu0lX9gpLhnF9nxlxEs,17494
+sws_spark_dissemination_helper/SWSEasyIcebergSparkHelper.py,sha256=csqKyYglBkJSBvEkEa1_keHarZZAIJHaV0d64gGJy98,26379
+sws_spark_dissemination_helper/SWSGoldIcebergSparkHelper.py,sha256=0dxbVkrhdaASapEffF5PFcgKwAMyJoWBxzgymjZ4JyY,25049
+sws_spark_dissemination_helper/SWSPostgresSparkReader.py,sha256=KpG8gp8Ai9pHDiKhUOTcXWxxmFGeKEE3XKlI_Y-SveU,18453
 sws_spark_dissemination_helper/SWSSilverIcebergSparkHelper.py,sha256=qioLv3SlJEfk0LzTiwfXRtZXVImPOJUeh9k1XwHC-pA,26225
 sws_spark_dissemination_helper/__init__.py,sha256=42TPbk7KxAud_qY3Sr_F4F7VjyofUlxEJkUXAFQsjRo,327
-sws_spark_dissemination_helper/constants.py,sha256=T-yTDS9spd0D9zv8Y2zIAU7Ie5MEt-IW0A1oeLm30mA,11413
+sws_spark_dissemination_helper/constants.py,sha256=vQmalAqInwPAybgJOfYx99jn47KsKp8jeD8eqmjw-Rs,13471
 sws_spark_dissemination_helper/utils.py,sha256=G7lQqNRrvqZpgm9WmddD7fWsI8IVn09x1p3cV3458EA,21963
-sws_spark_dissemination_helper-0.0.141.dist-info/METADATA,sha256=44O84wqYsAZ-pj5HdXkyuRVUEuBfRPZtzYsCAFwQgL0,2824
-sws_spark_dissemination_helper-0.0.141.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-sws_spark_dissemination_helper-0.0.141.dist-info/licenses/LICENSE,sha256=zFzeb_j_6pXEHwH8Z0OpIkKFJk7vmhZjdem-K0d4zU4,1073
-sws_spark_dissemination_helper-0.0.141.dist-info/RECORD,,
+sws_spark_dissemination_helper-0.0.171.dist-info/METADATA,sha256=W4qkQISSzekzXhpmNhlNMfJEmaQlscu3hQTs4Vavawg,2824
+sws_spark_dissemination_helper-0.0.171.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+sws_spark_dissemination_helper-0.0.171.dist-info/licenses/LICENSE,sha256=zFzeb_j_6pXEHwH8Z0OpIkKFJk7vmhZjdem-K0d4zU4,1073
+sws_spark_dissemination_helper-0.0.171.dist-info/RECORD,,